feat: Updating Python runtime to use Python SDK (#212)

* Adding initial version of the nodejs sdk.

* Adding pid info.

* Adding an example of using nodejs sdk.

* Adding test in examples for the new sdk.

* Simplifying package.json

* Cleaning up handler.

* chore: Adding NodeJS SDK to FEATURES.md & style fixes.

* Updating FEATURES.md

* fix: Fix for #190, broken nodejs sdk test.

* Re-eanbling the test.

* Trying out an example one more time.

* That's my output from running `make test-examples` locally:

```
--- PASS: Test (245.21s)
    --- PASS: Test/101-hello (10.63s)
    --- PASS: Test/101-two-node (2.79s)
    --- PASS: Test/102-filter (8.33s)
    --- PASS: Test/102-flatten-expand (8.69s)
    --- PASS: Test/102-map (11.33s)
    --- PASS: Test/103-autoscaling (11.03s)
    --- PASS: Test/103-scaling (12.76s)
    --- PASS: Test/104-golang1-16 (14.03s)
    --- PASS: Test/104-java16 (9.62s)
    --- PASS: Test/104-python3-9 (10.28s)
    --- PASS: Test/106-git-go (13.64s)
    --- PASS: Test/106-git-nodejs (13.74s)
    --- PASS: Test/106-git-python (32.50s)
    --- PASS: Test/107-completion (3.29s)
    --- PASS: Test/107-terminator (4.05s)
    --- PASS: Test/108-container (3.00s)
    --- PASS: Test/108-fifos (10.38s)
    --- PASS: Test/109-group (11.50s)
    --- PASS: Test/301-cron-log (9.87s)
    --- PASS: Test/301-http (13.50s)
    --- PASS: Test/301-kafka (10.30s)
    --- PASS: Test/301-two-sinks (8.23s)
    --- PASS: Test/301-two-sources (11.71s)
PASS
ok      github.com/argoproj-labs/argo-dataflow/examples 246.000s
```

Let's see if the same will happen in Github Actions.

* DO NOT MERGE: Testing out a different CI configuration to see if it will help with broken example tests.

* Adding NodeJS 16 Runtime & working NodeJS & Python example tests.

* Adding FEATURE.

* Updating example to point at main repo.

* fixing features order.

* Updating CONTRIBUTING docs.

* feat: Updating python runtime to use python SDK.

* Updating examples to use main repo code.

examples/git-python/handler.py

@@ -1,4 +1,4 @@
-def handler(myServer, message, empty):
+def handler(message, context):
   msg = message.decode("UTF-8")
   print('Got message', msg)
   return ("hi " + msg).encode('UTF-8')

runtimes/python3-9/main.py

@@ -1,32 +1,7 @@
-from http.server import BaseHTTPRequestHandler, HTTPServer
+from argo_dataflow_sdk import ProcessHandler
 
 from handler import handler
 
-
-class MyServer(BaseHTTPRequestHandler):
-    def do_GET(self):  # GET /ready
-        self.send_response(204)
-        self.end_headers()
-
-    def do_POST(self):  # POST /messages
-        len = int(self.headers.get('Content-Length'))
-        msg = self.rfile.read(len)
-        out = handler(msg, {})
-        if out:
-            self.send_response(201)
-            self.end_headers()
-            self.wfile.write(out)
-        else:
-            self.send_response(204)
-            self.end_headers()
-
-
 if __name__ == '__main__':
-    webServer = HTTPServer(("localhost", 8080), MyServer)
-
-    try:
-        webServer.serve_forever()
-    except KeyboardInterrupt:
-        pass
-
-    webServer.server_close()
+    processHandler = ProcessHandler()
+    processHandler.start(handler)

runtimes/python3-9/requirements.txt

@@ -1 +1,2 @@
-git+https://github.com/argoproj-labs/argo-dataflow#subdirectory=dsls/python
+git+https://github.com/argoproj-labs/argo-dataflow#subdirectory=dsls/python
+git+https://github.com/argoproj-labs/argo-dataflow#subdirectory=sdks/python

sdks/python/argo_dataflow_sdk/main.py

@@ -6,21 +6,22 @@
 hostName = "0.0.0.0"
 serverPort = 8080
 threads = set()
+handler_ref = None
 
 class MyServer(BaseHTTPRequestHandler):
-  handler = None
   def do_GET(self):  # GET /ready
     threads.add(threading.currentThread().getName())
     self.send_response(204)
     self.end_headers()
     threads.remove(threading.currentThread().getName())
 
   def do_POST(self):  # POST /messages
+    global handler_ref
     try:
       threads.add(threading.currentThread().getName())
       len = int(self.headers.get('Content-Length'))
       msg = self.rfile.read(len)
-      out = self.handler(msg, {})
+      out = handler_ref(msg, {})
       if out:
         self.send_response(201)
         self.end_headers()
@@ -30,7 +31,7 @@ def do_POST(self):  # POST /messages
         self.end_headers()
     except Exception as err:
       exception_type = type(err).__name__
-      print(exception_type)
+      print(exception_type, err)
       self.send_response(500)
       self.end_headers()
       self.wfile.write(err.__str__().encode('UTF-8'))
@@ -54,8 +55,9 @@ def terminate(self, signal, frame):
     sys.exit(0)
 
   def start(self, handler):
+    global handler_ref
     signal.signal(signal.SIGTERM, self.terminate)
-    MyServer.handler = handler
+    handler_ref = handler
     self.webServer = ThreadingHTTPServer((hostName, serverPort), MyServer)
     print("Server started http://%s:%s with pid %i" % (hostName, serverPort, os.getpid()))
     try:
@@ -69,12 +71,12 @@ def start(self, handler):
     self.webServer.server_close()
     sys.exit(0)
 
-def default_handler(myServer, message, empty):
+def default_handler(message, context):
   msg = message.decode("UTF-8")
   print('Got message', msg)
   return ("Hi " + msg).encode('UTF-8')
 
-def default_handler_error(myServer, message, empty):
+def default_handler_error(message, context):
   msg = message.decode("UTF-8")
   print('Got message', msg)
   raise ValueError('Some error')

sdks/python/setup.py

@@ -4,7 +4,7 @@
     name='argo_dataflow_sdk',
     packages=['argo_dataflow_sdk'],
     install_requires=[],
-    version='v0.0.76',
+    version='v0.0.84',
     license='apache-2.0',
     description='Argo Dataflow SDK. Can be used to fulfill Argo-Dataflow\'s IMAGE CONTRACT: https://github.com/argoproj-labs/argo-dataflow/blob/main/docs/IMAGE_CONTRACT.md',
     author='Dom Deren',