From 186b9c964190bcc71f90856f0ae966be30e9f1b8 Mon Sep 17 00:00:00 2001 From: Evgeny Skvortsov <71790359+EvgSkv@users.noreply.github.com> Date: Tue, 18 Jul 2023 18:44:58 -0400 Subject: [PATCH] Support ascii graph in Jupyter. --- colab_logica.py | 41 +++++- common/concertina_lib.py | 35 ++++- compiler/dialect_libraries/psql_library.py | 2 + examples/more/Simple_Postgres.ipynb | 138 +++++++++++++++++++ logica.py | 15 +- tools/run_in_terminal.py | 26 +++- type_inference/research/types_of_builtins.py | 4 + 7 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 examples/more/Simple_Postgres.ipynb diff --git a/colab_logica.py b/colab_logica.py index 24f86a4..cb23c85 100755 --- a/colab_logica.py +++ b/colab_logica.py @@ -16,6 +16,9 @@ """Library for using Logica in CoLab.""" +import getpass +import json + from .common import color from .common import concertina_lib @@ -23,6 +26,8 @@ from .compiler import rule_translate from .compiler import universe +from .type_inference.research import infer + import IPython from IPython.core.magic import register_cell_magic @@ -70,6 +75,8 @@ PREAMBLE = None +DISPLAY_MODE = 'colab' # or colab-text + def SetPreamble(preamble): global PREAMBLE @@ -83,6 +90,21 @@ def SetDbConnection(connection): global DB_CONNECTION DB_CONNECTION = connection +def ConnectToPostgres(mode='interactive'): + import psycopg2 + if mode == 'interactive': + print('Please enter PostgreSQL connection config in JSON format.') + print('Example:') + print('{"host": "myhost", "database": "megadb", ' + '"user": "myuser", "password": "42"}') + connection_str = getpass.getpass() + elif mode == 'environment': + connection_str = os.environ.get('LOGICA_PSQL_CONNECTION') + else: + assert False, 'Unknown mode:' + mode + connection_json = json.loads(connection_str) + SetDbConnection(psycopg2.connect(**connection_json)) + def EnsureAuthenticatedUser(): global USER_AUTHENTICATED global PROJECT @@ -142,12 +164,17 @@ def RunSQL(sql, engine, connection=None, is_final=False): client = bigquery.Client(project=PROJECT) return client.query(sql).to_dataframe() elif engine == 'psql': - # Sorry, this is not looking good. - from sqlalchemy import text if is_final: - return pandas.read_sql(text(sql), connection) + cursor = connection.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + df = pandas.DataFrame( + rows, columns=[d[0] for d in cursor.description]) + return df else: - return connection.execute(text(sql)) + cursor = connection.cursor() + cursor.execute(sql) + connection.commit() elif engine == 'sqlite': try: if is_final: @@ -215,6 +242,9 @@ def Logica(line, cell, run_query): except rule_translate.RuleCompileException as e: e.ShowMessage() return + except infer.TypeErrorCaughtException as e: + e.ShowMessage() + return engine = program.annotations.Engine() @@ -273,7 +303,8 @@ def Logica(line, cell, run_query): 'for now.') result_map = concertina_lib.ExecuteLogicaProgram( - executions, sql_runner=sql_runner, sql_engine=engine) + executions, sql_runner=sql_runner, sql_engine=engine, + display_mode=DISPLAY_MODE) for idx, predicate in enumerate(predicates): t = result_map[predicate] diff --git a/common/concertina_lib.py b/common/concertina_lib.py index 94a9674..54f5d1f 100644 --- a/common/concertina_lib.py +++ b/common/concertina_lib.py @@ -4,10 +4,15 @@ try: import graphviz +except: + print('Could not import graphviz tools in Concertina.') + +try: + from IPython.display import HTML from IPython.display import display from IPython.display import update_display except: - print('Could not import CoLab tools in Concertina.') + print('Could not import IPython in Concertina.') if '.' not in __package__: from common import graph_art @@ -75,7 +80,7 @@ def __init__(self, config, engine, display_mode='colab'): self.all_actions = {a["name"] for a in self.config} self.complete_actions = set() self.running_actions = set() - assert display_mode in ('colab', 'terminal'), ( + assert display_mode in ('colab', 'terminal', 'colab-text'), ( 'Unrecognized display mode: %s' % display_mode) self.display_mode = display_mode self.display_id = self.GetDisplayId() @@ -137,7 +142,14 @@ def AsNodesAndEdges(self): """Nodes and edges to display in terminal.""" def ColoredNode(node): if node in self.running_actions: - return '\033[1m\033[93m' + node + '\033[0m' + if self.display_mode == 'terminal': + return '\033[1m\033[93m' + node + '\033[0m' + elif self.display_mode == 'colab-text': + return ( + '' + node + '' + ) + else: + assert False, self.display_mode else: return node nodes = [] @@ -150,11 +162,24 @@ def ColoredNode(node): edges.append([prerequisite_node, a_node]) return nodes, edges + def StateAsSimpleHTML(self): + style = ';'.join([ + 'border: 1px solid rgba(0, 0, 0, 0.3)', + 'width: fit-content;', + 'padding: 20px', + 'border-radius: 5px', + 'box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.2)']) + return HTML('
%s
' % ( + style, self.AsTextPicture(updating=False))) + def Display(self): if self.display_mode == 'colab': display(self.AsGraphViz(), display_id=self.display_id) elif self.display_mode == 'terminal': print(self.AsTextPicture(updating=False)) + elif self.display_mode == 'colab-text': + display(self.StateAsSimpleHTML(), + display_id=self.display_id) else: assert 'Unexpected mode:', self.display_mode @@ -163,6 +188,10 @@ def UpdateDisplay(self): update_display(self.AsGraphViz(), display_id=self.display_id) elif self.display_mode == 'terminal': print(self.AsTextPicture(updating=True)) + elif self.display_mode == 'colab-text': + update_display( + self.StateAsSimpleHTML(), + display_id=self.display_id) else: assert 'Unexpected mode:', self.display_mode diff --git a/compiler/dialect_libraries/psql_library.py b/compiler/dialect_libraries/psql_library.py index 2bd1e31..0e80cfd 100644 --- a/compiler/dialect_libraries/psql_library.py +++ b/compiler/dialect_libraries/psql_library.py @@ -37,4 +37,6 @@ "ARRAY_AGG({value} order by {arg})", {arg: a.arg, value: a.value}); +RecordAsJson(r) = SqlExpr( + "ROW_TO_JSON({r})", {r:}); """ diff --git a/examples/more/Simple_Postgres.ipynb b/examples/more/Simple_Postgres.ipynb new file mode 100644 index 0000000..4a96aec --- /dev/null +++ b/examples/more/Simple_Postgres.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "7e89b446", + "metadata": {}, + "source": [ + "# Example of running Postgres with text Pipeline overview" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "58f2b0db", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "from logica import colab_logica\n", + "colab_logica.ConnectToPostgres('interactive')\n", + "colab_logica.DISPLAY_MODE = 'colab-text'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "82c02eb6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Query is stored at \u001b[1mGreeting_sql\u001b[0m variable.\n" + ] + }, + { + "data": { + "text/html": [ + "
 ▚  Greeting
" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Running predicate: Greeting (0 seconds)\n", + "The following table is stored at \u001b[1mGreeting\u001b[0m variable.\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
col0
0Hello world!
\n", + "
" + ], + "text/plain": [ + " col0\n", + "0 Hello world!" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " \n" + ] + } + ], + "source": [ + "%%logica Greeting\n", + "\n", + "@Engine(\"psql\");\n", + "\n", + "Greeting(\"Hello world!\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/logica.py b/logica.py index 46a508c..604baaa 100755 --- a/logica.py +++ b/logica.py @@ -139,6 +139,7 @@ def main(argv): print('Not enough arguments. Run \'logica help\' for help.', file=sys.stderr) return 1 + predicates = argv[3] if argv[1] == '-': filename = '/dev/stdin' @@ -158,6 +159,13 @@ def main(argv): if not os.path.exists(filename): print('File not found: %s' % filename, file=sys.stderr) return 1 + + if command == 'run_in_terminal': + from tools import run_in_terminal + artistic_table = run_in_terminal.Run(filename, predicates) + print(artistic_table) + return + program_text = open(filename).read() try: @@ -187,8 +195,6 @@ def main(argv): type_error_checker.CheckForError() return 0 - predicates = argv[3] - user_flags = ReadUserFlags(parsed_rules, argv[4:]) predicates_list = predicates.split(',') @@ -265,11 +271,6 @@ def main(argv): assert False, 'Unknown engine: %s' % engine print(o.decode()) - if command == 'run_in_terminal': - from tools import run_in_terminal - artistic_table = run_in_terminal.Run(filename, predicate) - print(artistic_table) - def run_main(): """Run main function with system arguments.""" diff --git a/tools/run_in_terminal.py b/tools/run_in_terminal.py index fe3ee98..8a20c8b 100644 --- a/tools/run_in_terminal.py +++ b/tools/run_in_terminal.py @@ -16,6 +16,9 @@ # Utility to run pipeline in terminal with ASCII art showing progress. +import json +import os + if not __package__ or '.' not in __package__: from common import concertina_lib from compiler import universe @@ -31,7 +34,7 @@ class SqlRunner(object): def __init__(self, engine): self.engine = engine - assert engine in ['sqlite', 'bigquery'] + assert engine in ['sqlite', 'bigquery', 'psql'] if engine == 'sqlite': self.connection = sqlite3_logica.SqliteConnect() else: @@ -45,6 +48,16 @@ def __init__(self, engine): credentials, project = auth.default() else: credentials, project = None, None + if engine == 'psql': + import psycopg2 + if os.environ.get('LOGICA_PSQL_CONNECTION'): + connection_json = json.loads(os.environ.get('LOGICA_PSQL_CONNECTION')) + else: + assert False, ( + 'Please provide PSQL connection parameters ' + 'in LOGICA_PSQL_CONNECTION') + self.connection = psycopg2.connect(**connection_json) + self.bq_credentials = credentials self.bq_project = project @@ -68,10 +81,17 @@ def RunSQL(sql, engine, connection=None, is_final=False, elif engine == 'psql': import pandas if is_final: - df = pandas.read_sql(sql, connection) + cursor = connection.cursor() + cursor.execute(sql) + rows = cursor.fetchall() + df = pandas.DataFrame( + rows, columns=[d[0] for d in cursor.description]) + connection.close() return list(df.columns), [list(r) for _, r in df.iterrows()] else: - return connection.execute(sql) + cursor = connection.cursor() + cursor.execute(sql) + connection.commit() elif engine == 'sqlite': try: if is_final: diff --git a/type_inference/research/types_of_builtins.py b/type_inference/research/types_of_builtins.py index e35d490..88a6b12 100644 --- a/type_inference/research/types_of_builtins.py +++ b/type_inference/research/types_of_builtins.py @@ -127,6 +127,10 @@ def TypesOfBultins(): 'ValueOfUnnested': { 0: x, 'logica_value': x + }, + 'RecordAsJson': { + 0: reference_algebra.OpenRecord({}), + 'logica_value': 'Str' } } return {