Changed docker images for db and dummy data

Author: ClaudioPrimerano

database/Dockerfile

@@ -1,3 +1,3 @@
-FROM timescale/timescaledb-ha:pg16.1-ts2.13.1-all
+FROM postgis/postgis:16-3.4-alpine
 COPY ./istsos_schema.sql /docker-entrypoint-initdb.d/11_init_db_istsos_schema.sql
 COPY ./istsos_schema_versioning.sql /docker-entrypoint-initdb.d/12_init_db_istsos_schema_versioning.sql

database/istsos_schema.sql

@@ -1,9 +1,5 @@
---CREATE EXTENSION IF NOT exists pg_graphql;
 CREATE EXTENSION IF NOT exists postgis;
-CREATE EXTENSION IF NOT exists unit;
-CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
 CREATE EXTENSION IF NOT EXISTS btree_gist;
---CREATE EXTENSION IF NOT exists uri;
 
 CREATE SCHEMA sensorthings;
 

dummy_data/Dockerfile

@@ -1,5 +1,5 @@
 # Use an official Python runtime as a parent image
-FROM python:3.10
+FROM python:3.12.4-alpine3.20
 
 # Set the working directory to /app
 WORKDIR /dummy_data

dummy_data/generator.py

@@ -2,9 +2,10 @@
 import os
 import random
 from datetime import datetime, time
+import asyncio
 
+import asyncpg
 import isodate
-import psycopg2
 
 create_dummy_data = int(os.getenv("DUMMY_DATA"))
 delete_dummy_data = int(os.getenv("CLEAR_DATA"))
@@ -18,14 +19,25 @@
     else datetime.combine(datetime.now().today(), time.min)
 )
 
-connection_url = "dbname={db} user={user} password={password} host='database' port='5432'".format(
-    db=os.getenv("POSTGRES_DB"),
-    user=os.getenv("POSTGRES_USER"),
-    password=os.getenv("POSTGRES_PASSWORD"),
-)
+pgpool = None
+
+async def get_pool():
+    """
+    Retrieves or creates a connection pool to the PostgreSQL database.
+
+    Returns:
+        asyncpg.pool.Pool: The connection pool object.
+    """
+    global pgpool
+    if not pgpool:
+        pgpool = await asyncpg.create_pool(
+            dsn=f"postgresql://{os.getenv("POSTGRES_USER")}:{os.getenv("POSTGRES_PASSWORD")}@database:5432/{os.getenv("POSTGRES_DB")}"
+        )
+    return pgpool
+
 
 
-def generate_things(cur):
+async def generate_things(conn):
     """
     Generate a list of things and insert them into the database.
 
@@ -44,12 +56,12 @@ def generate_things(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Thing" (description, name, properties)
-    VALUES (%s, %s, %s)
+    VALUES ($1, $2, $3)
     """
-    cur.executemany(insert_sql, things)
+    await conn.executemany(insert_sql, things)
 
 
-def generate_locations(cur):
+async def generate_locations(conn):
     """
     Generate locations and insert them into the database.
 
@@ -69,12 +81,12 @@ def generate_locations(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Location" (description, name, location, "encodingType")
-    VALUES (%s, %s, %s, %s)
+    VALUES ($1, $2, $3, $4)
     """
-    cur.executemany(insert_sql, locations)
+    await conn.executemany(insert_sql, locations)
 
 
-def generate_things_locations(cur):
+async def generate_things_locations(conn):
     """
     Generate and insert things locations into the database.
 
@@ -92,12 +104,12 @@ def generate_things_locations(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Thing_Location" (thing_id, location_id)
-    VALUES (%s, %s)
+    VALUES ($1, $2)
     """
-    cur.executemany(insert_sql, things_locations)
+    await conn.executemany(insert_sql, things_locations)
 
 
-def generate_historicallocations(cur):
+async def generate_historicallocations(conn):
     """
     Generate historical locations for things and insert them into the database.
 
@@ -116,12 +128,12 @@ def generate_historicallocations(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."HistoricalLocation" (time, thing_id)
-    VALUES (%s, %s)
+    VALUES ($1, $2)
     """
-    cur.executemany(insert_sql, historicallocations)
+    await conn.executemany(insert_sql, historicallocations)
 
 
-def generate_locations_historicallocations(cur):
+async def generate_locations_historicallocations(conn):
     """
     Generate a list of tuples representing the relationship between location and historical location.
 
@@ -141,12 +153,12 @@ def generate_locations_historicallocations(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Location_HistoricalLocation" (location_id, historicallocation_id)
-    VALUES (%s, %s)
+    VALUES ($1, $2)
     """
-    cur.executemany(insert_sql, locations_historicallocations)
+    await conn.executemany(insert_sql, locations_historicallocations)
 
 
-def generate_observedProperties(cur):
+async def generate_observedProperties(conn):
     """
     Generate observed properties and insert them into the database.
 
@@ -165,12 +177,12 @@ def generate_observedProperties(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."ObservedProperty" (name, definition, description)
-    VALUES (%s, %s, %s)
+    VALUES ($1, $2, $3)
     """
-    cur.executemany(insert_sql, observedProperties)
+    await conn.executemany(insert_sql, observedProperties)
 
 
-def generate_sensors(cur):
+async def generate_sensors(conn):
     """
     Generate a list of sensors with random descriptions, names, encoding types, and metadata.
 
@@ -190,12 +202,12 @@ def generate_sensors(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Sensor" (description, name, "encodingType", metadata)
-    VALUES (%s, %s, %s, %s)
+    VALUES ($1, $2, $3, $4)
     """
-    cur.executemany(insert_sql, sensors)
+    await conn.executemany(insert_sql, sensors)
 
 
-def generate_datastreams(cur):
+async def generate_datastreams(conn):
     """
     Generate datastreams and insert them into the database.
 
@@ -237,12 +249,12 @@ def generate_datastreams(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Datastream" ("unitOfMeasurement", description, name, "observationType", thing_id, sensor_id, observedproperty_id)
-    VALUES (%s, %s, %s, %s, %s, %s, %s)
+    VALUES ($1, $2, $3, $4, $5, $6, $7)
     """
-    cur.executemany(insert_sql, datastreams)
+    await conn.executemany(insert_sql, datastreams)
 
 
-def generate_featuresofinterest(cur):
+async def generate_featuresofinterest(conn):
     """
     Generate features of interest and insert them into the database.
 
@@ -262,12 +274,12 @@ def generate_featuresofinterest(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."FeaturesOfInterest" (description, name, "encodingType", feature)
-    VALUES (%s, %s, %s, %s)
+    VALUES ($1, $2, $3, $4)
     """
-    cur.executemany(insert_sql, featuresofinterest)
+    await conn.executemany(insert_sql, featuresofinterest)
 
 
-def generate_observations(cur):
+async def generate_observations(conn):
     """
     Generates observations and inserts them into the database.
 
@@ -298,12 +310,12 @@ def generate_observations(cur):
 
     insert_sql = """
     INSERT INTO sensorthings."Observation" ("phenomenonTime", "resultInteger", "resultType", datastream_id, featuresofinterest_id)
-    VALUES (%s, %s, %s, %s, %s)
+    VALUES ($1, $2, $3, $4, $5)
     """
-    cur.executemany(insert_sql, observations)
+    await conn.executemany(insert_sql, observations)
 
 
-def create_data():
+async def create_data():
     """
     Generates dummy data and inserts it into the database.
 
@@ -314,29 +326,25 @@ def create_data():
 
     After the creation is complete, the database connection is closed.
     """
-    conn = psycopg2.connect(connection_url)
-    cur = conn.cursor()
-    try:
-        generate_things(cur)
-        generate_locations(cur)
-        generate_things_locations(cur)
-        generate_historicallocations(cur)
-        generate_locations_historicallocations(cur)
-        generate_observedProperties(cur)
-        generate_sensors(cur)
-        generate_datastreams(cur)
-        generate_featuresofinterest(cur)
-        generate_observations(cur)
-        conn.commit()
-    except Exception as e:
-        conn.rollback()
-        print(f"An error occurred: {e}")
-    finally:
-        cur.close()
-        conn.close()
-
-
-def delete_data():
+    pool = await get_pool()
+    async with pool.acquire() as conn:
+        async with conn.transaction():
+            try:
+                await generate_things(conn)
+                await generate_locations(conn)
+                await generate_things_locations(conn)
+                await generate_historicallocations(conn)
+                await generate_locations_historicallocations(conn)
+                await generate_observedProperties(conn)
+                await generate_sensors(conn)
+                await generate_datastreams(conn)
+                await generate_featuresofinterest(conn)
+                await generate_observations(conn)
+            except Exception as e:
+                print(f"An error occured: {e}")
+
+
+async def delete_data():
     """
     Deletes all data from the sensorthings tables in the database.
 
@@ -358,30 +366,28 @@ def delete_data():
 
     After the deletion is complete, the database connection is closed.
     """
-    conn = psycopg2.connect(connection_url)
-    cur = conn.cursor()
-    try:
-        cur.execute('DELETE FROM sensorthings."Thing"')
-        cur.execute('DELETE FROM sensorthings."Location"')
-        cur.execute('DELETE FROM sensorthings."Thing_Location"')
-        cur.execute('DELETE FROM sensorthings."HistoricalLocation"')
-        cur.execute('DELETE FROM sensorthings."Location_HistoricalLocation"')
-        cur.execute('DELETE FROM sensorthings."ObservedProperty"')
-        cur.execute('DELETE FROM sensorthings."Sensor"')
-        cur.execute('DELETE FROM sensorthings."Datastream"')
-        cur.execute('DELETE FROM sensorthings."FeaturesOfInterest"')
-        cur.execute('DELETE FROM sensorthings."Observation"')
-        conn.commit()
-    except Exception as e:
-        conn.rollback()
-        print(f"An error occurred: {e}")
-    finally:
-        cur.close()
-        conn.close()
+    pool = await get_pool()
+    async with  pool.acquire() as conn:
+        async with conn.transaction():
+            try:
+                await conn.execute('DELETE FROM sensorthings."Thing"')
+                await conn.execute('DELETE FROM sensorthings."Location"')
+                await conn.execute('DELETE FROM sensorthings."Thing_Location"')
+                await conn.execute('DELETE FROM sensorthings."HistoricalLocation"')
+                await conn.execute('DELETE FROM sensorthings."Location_HistoricalLocation"')
+                await conn.execute('DELETE FROM sensorthings."ObservedProperty"')
+                await conn.execute('DELETE FROM sensorthings."Sensor"')
+                await conn.execute('DELETE FROM sensorthings."Datastream"')
+                await conn.execute('DELETE FROM sensorthings."FeaturesOfInterest"')
+                await conn.execute('DELETE FROM sensorthings."Observation"')
+            except Exception as e:
+                print(f"An error occurred: {e}")
+            finally:
+                await conn.close()
 
 
 if __name__ == "__main__":
     if delete_dummy_data:
-        delete_data()
+        asyncio.run(delete_data())
     if create_dummy_data:
-        create_data()
+        asyncio.run(create_data())

dummy_data/requirements.txt

@@ -1,4 +1,3 @@
-psycopg2
-PyYAML
 python-dotenv
-isodate
+isodate
+asyncpg