started data connection #2

Signed-off-by: Alejandro Ouslan <alejandro.ouslan@upr.edu>

Author: ouslan

.dockerignore

@@ -1,4 +1,4 @@
-notebooks/
+earnotebooks/
 
 # Ignore all pkl files
 *.pkl
@@ -14,7 +14,11 @@ notebooks/
 data/raw/*.csv.gz
 data/shape_files/*.zip
 *.csv.gz
-*.zip
+
+# Exceptions
+!data/shape_files/states.zip
+!data/processed/blocks.parquet
+!data/processed/lodes.parquet
 
 # Ignore Python bytecode files
 *.pyc

docker-compose.yaml

@@ -36,7 +36,7 @@ services:
       networks:
         - timescale_network
   pgadmin:
-      image: dpage/pgadmin4:4.23
+      image: dpage/pgadmin4:8.8
       container_name: dev_pgadmin4
       environment:
         PGADMIN_DEFAULT_EMAIL: ${PGADMIN_EMAIL}

docs/conclusion.md

@@ -11,7 +11,9 @@ dependencies:
   - numpy=1.26.4
   - ipykernel=6.29.3
   - dash=2.17.0
-  #- plotly-geo=1.0.0
+  - pyarrow=16.1.0
+  - psycopg2=2.9.9
   - pip:
       - geoarrow-rust-core==0.2.0
+      - python-dotenv==1.0.1
       - pyogrio==0.8.0

docs/literature.md

@@ -0,0 +1,7 @@
+from src.data.data_process import DataProcess
+
+def main() -> None:
+    DataProcess()
+
+if __name__ == "__main__":
+    main()

environment.yml

@@ -7,4 +7,6 @@ geoarrow-rust-core==0.2.0
 pyogrio==0.8.0
 geopandas==0.14.4
 pyarrow==16.1.0
-gunicorn==22.0.0
+gunicorn==22.0.0
+psycopg2==2.9.9
+python-dotenv==1.0.1

main.py

@@ -1,5 +1,5 @@
-import psycopg2
 import os
+import psycopg2
 from dotenv import load_dotenv
 load_dotenv()
 
@@ -14,28 +14,41 @@ def __init__(self):
              f"host={df_port} " 
              f"port={df_port}")
         self.conn = psycopg2.connect(connection_url)
+        self.create_state_tables()
 
     def create_state_tables(self):
         cursor = self.conn.cursor()
         query = """CREATE TABLE IF NOT EXISTS states(
-                       id serial primary key,
+                       state_id serial primary key,
+                       state_num varchar(2) NOT NULL,
                        state_abbr varchar(2) NOT NULL,
                        state_name varchar(100) NOT NULL,
                        geom geometry NOT NULL
                     );
                 """
-        cursor.execute(query,)
+        cursor.execute(query)
         self.conn.commit()
 
-    def create_block_tables(self):
+    def create_blocks_tables(self):
         cursor = self.conn.cursor()
         query = """CREATE TABLE IF NOT EXISTS blocks(
                        block_id serial primary key,                    
                        states foreign key (state_name) references states(state_name),
                        geom geometry NOT NULL
                     );
                 """
-        cursor.execute(query,)
+        cursor.execute(query)
+        self.conn.commit()
+    
+    def create_puma_tables(self):
+        cursor = self.conn.cursor()
+        query = """CREATE TABLE IF NOT EXISTS pumas(
+                       puma serial primary key,                    
+                       states foreign key (state_name) references states(state_name),
+                       geom geometry NOT NULL
+                    );
+                """
+        cursor.execute(query)
         self.conn.commit()
 
     def create_lodes_tables(self):
@@ -44,15 +57,12 @@ def create_lodes_tables(self):
                        state_name foreign key (state_name) references states(state_name),
                    );
                 """
-        cursor.execute(query,)
+        cursor.execute(query)
         self.conn.commit()
 
     def create_distance_tables(self):
         pass 
 
-    def create_distance_tables(self):
-        pass 
-
     def create_movs_tables(self):
         cursor = self.conn.cursor()
         query = """CREATE TABLE IF NOT EXISTS ledes(
@@ -63,40 +73,17 @@ def create_movs_tables(self):
                     net_value FLOAT NOT NULL
             );
             """
-        cursor.execute(query,)
+        cursor.execute(query)
         self.conn.commit()
 
-    def create_distance_tables(self):
-        pass 
-
     def create_movs_tables(self):
         cursor = self.conn.cursor()
         query = """CREATE TABLE IF NOT EXISTS ledes(
                     block_id serial primary key,
                     state_name foreign key (state_name) references states(state_name),
             );
             """
-        cursor.execute(query,)
-        self.conn.commit()
-    
-    def create_sex_tables(self):
-        pass
-
-    def create_race_tables(self):
-        pass
-    
-
-    def create_distance_tables(self):
-        pass 
-
-    def create_movs_tables(self):
-        cursor = self.conn.cursor()
-        query = """CREATE TABLE IF NOT EXISTS ledes(
-                    block_id serial primary key,
-                    state_name foreign key (state_name) references states(state_name),
-            );
-            """
-        cursor.execute(query,)
+        cursor.execute(query)
         self.conn.commit()
 
     def create_sex_tables(self):
@@ -107,12 +94,12 @@ def create_race_tables(self):
 
     def create_hypertable(self):
         cursor = self.conn.commit()
-        query = """select create_hypertable(
-            'ledes',
-            interval '1 year'
-        )
+        query = """SELECT create_hypertable(
+                      'ledes',
+                       interval '1 year'
+            );
         """
-        cursor.execute(query,)
+        cursor.execute(query)
         self.conn.commit()
 
 if __name__ == "__main__":

requirements.txt

@@ -15,7 +15,6 @@ def load_shape_data(self):
         return shp
 
     def create_graph_dataset(self) -> gpd.GeoDataFrame:
-
         df = self.data.rename({"state": "STUSPS"})
         df = df.with_columns(pl.col("STUSPS").str.to_uppercase())
         df = df.to_pandas()