From db0cc46a71d007bde4c223b37c30dbcd5c1b43ba Mon Sep 17 00:00:00 2001 From: Nikoletos Konstantinos <47646955+Nikoletos-K@users.noreply.github.com> Date: Fri, 30 Jun 2023 15:14:24 +0300 Subject: [PATCH] changed repo structure --- .github/workflows/python-publish.yml | 39 ------ .gitignore | 1 + about.md | 34 ----- docs-only/block_cleaning.rst | 5 - docs-only/clustering.rst | 5 - docs-only/comparison_cleaning.rst | 2 - docs-only/datamodel.rst | 5 - docs-only/evaluation.rst | 5 - docs-only/intersphinx.md | 28 ---- docs-only/matching.rst | 5 - docs-only/similarity_joins.rst | 6 - docs-only/utils.rst | 5 - {documentation => docs/img}/3DERslide.png | Bin .../img}/demo-architecture.png | Bin {documentation => docs/img}/logo_only.png | Bin {documentation => docs/img}/pLibTool.PNG | Bin .../img}/pyJedAI-768x772.jpg | Bin .../img}/pyjedai-logo-lin.png | Bin {documentation => docs/img}/pyjedai-ppt.png | Bin .../img}/pyjedai.logo.drawio.png | Bin .../img}/pyjedai.logo.drawio.sm.png | Bin .../img}/reading-process.jpg | Bin .../img}/reading-process.png | Bin .../img}/workflow-example.png | Bin .../img}/workflow1-cora.png | Bin {documentation => docs/img}/workflow1.png | Bin .../img}/workflow2-cora.png | Bin {documentation => docs/img}/workflow2.png | Bin {documentation => docs/img}/workflow3.png | Bin index.md | 131 ------------------ pyproject.toml | 5 +- 31 files changed, 5 insertions(+), 271 deletions(-) delete mode 100644 .github/workflows/python-publish.yml delete mode 100644 about.md delete mode 100644 docs-only/block_cleaning.rst delete mode 100644 docs-only/clustering.rst delete mode 100644 docs-only/comparison_cleaning.rst delete mode 100644 docs-only/datamodel.rst delete mode 100644 docs-only/evaluation.rst delete mode 100644 docs-only/intersphinx.md delete mode 100644 docs-only/matching.rst delete mode 100644 docs-only/similarity_joins.rst delete mode 100644 docs-only/utils.rst rename {documentation => docs/img}/3DERslide.png (100%) rename {documentation => docs/img}/demo-architecture.png (100%) rename {documentation => docs/img}/logo_only.png (100%) rename {documentation => docs/img}/pLibTool.PNG (100%) rename {documentation => docs/img}/pyJedAI-768x772.jpg (100%) rename {documentation => docs/img}/pyjedai-logo-lin.png (100%) rename {documentation => docs/img}/pyjedai-ppt.png (100%) rename {documentation => docs/img}/pyjedai.logo.drawio.png (100%) rename {documentation => docs/img}/pyjedai.logo.drawio.sm.png (100%) rename {documentation => docs/img}/reading-process.jpg (100%) rename {documentation => docs/img}/reading-process.png (100%) rename {documentation => docs/img}/workflow-example.png (100%) rename {documentation => docs/img}/workflow1-cora.png (100%) rename {documentation => docs/img}/workflow1.png (100%) rename {documentation => docs/img}/workflow2-cora.png (100%) rename {documentation => docs/img}/workflow2.png (100%) rename {documentation => docs/img}/workflow3.png (100%) delete mode 100644 index.md diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index be63051..0000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: PYJEDAI_PYPI_TOKEN - password: ${{ secrets.PYJEDAI_PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore index 3183829..3371fe3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ examples/scripts/.ipynb_checkpoints/ tests/__pycache__/ .hypothesis/unicode_data/11.0.0/charmap.json.gz src/pyjedai/__pycache__/ +src/pyjedai.egg-info/* diff --git a/about.md b/about.md deleted file mode 100644 index e87201e..0000000 --- a/about.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -layout: page -title: Overview ---- - - -
-pyJedAI-Architecture - -
- -
- -# Architecture - -The purpose of this framework is to demonstrate how ER can be accomplished by expert and novice users in an intuitive, yet efficient and effective way. pyJedai addresses the following task: - -Given a source and a target dataset, S and T, respectively, discover the set of links - -

L = \{(s,owl:sameAS, t) | s \epsilon S ∧ t \epsilon T\}

- -Its architecture appears in the bellow figure. The first module is the data reader, which specifies the user input. pyJedAI supports both semi-structured and structured data as input. The former, which include SPARQL endpoints and RDF/OWL dumps, are read by RDFLib. The latter, which include relational databases as well as CSV and JSON files, are read by pandas. In this way, pyJedAI is able to interlink any combination of semi-structured and structured data sources, which is a unique feature. The second step in pyJedAI’s pipeline performs block building, a coarsegrained process that clusters together similar entities. The end result consists of a set of candidate pairs, which are examined analytically by the subsequent steps. pyJedAI implements the same established methods for similarity joins and blocking as JedAI, such as Standard Blocking and Sorted Neighborhood, but goes beyond all Link Discovery tools by incorporating recent, state-of-the-art libraries for nearest neighbor search like FALCONN and FAISS. - -
- -
-pyJedAI-Architecture -
- -
- - The entity matching step estimates the actual similarity between the candidate pairs. Unlike all other Link Discovery tools, which rely exclusively on string similarity measures like edit distance and Jaccard coefficient, pyJedAI leverages the latest advanced NLP techniques, like pre-trained embeddings (e.g., word2vect, fastText and Glove) and transformer language models (i.e., BERT and its variants). More specifically, pyJedAI supports packages like strsimpy, Gensimand Hugging Face. This unique feature boosts pyJedAI’s accuracy to a significant extent, without requiring any labelled instances from the user. The last step performs entity clustering to further increase the accuracy. The relevant techniques consider the global information provided by the similarity scores of all candidate pairs in order to take local decisions for each pair of entity descriptions. pyJedAI implements and offers the same established algorithms as JedAI, using NetworkX to ensure high time efficiency. Finally, users are able to evaluate, visualize and store the results of the selected pipeline through the intuitive interface of Jupyter notebooks. In this way, pyJedAI facilitates its use by researchers and practitioners that are familiar with the data science ecosystem, regardless of their familiarity with ER and Link -Discovery, in general. - diff --git a/docs-only/block_cleaning.rst b/docs-only/block_cleaning.rst deleted file mode 100644 index 377fb01..0000000 --- a/docs-only/block_cleaning.rst +++ /dev/null @@ -1,5 +0,0 @@ -Block Cleaning - block_cleaning.py -================================== - -.. .. automodule:: pyjedai.block_cleaning -.. :members: \ No newline at end of file diff --git a/docs-only/clustering.rst b/docs-only/clustering.rst deleted file mode 100644 index ae8a941..0000000 --- a/docs-only/clustering.rst +++ /dev/null @@ -1,5 +0,0 @@ -Clustering - clustering.py -============================ - -.. .. automodule:: pyjedai.clustering -.. :members: \ No newline at end of file diff --git a/docs-only/comparison_cleaning.rst b/docs-only/comparison_cleaning.rst deleted file mode 100644 index 8805a41..0000000 --- a/docs-only/comparison_cleaning.rst +++ /dev/null @@ -1,2 +0,0 @@ -Comparison Cleaning -=================== \ No newline at end of file diff --git a/docs-only/datamodel.rst b/docs-only/datamodel.rst deleted file mode 100644 index 8cc7cdd..0000000 --- a/docs-only/datamodel.rst +++ /dev/null @@ -1,5 +0,0 @@ -Datamodel - datamodel.py -======================== - -.. .. automodule:: pyjedai.datamodel -.. :members: \ No newline at end of file diff --git a/docs-only/evaluation.rst b/docs-only/evaluation.rst deleted file mode 100644 index 0ac1c07..0000000 --- a/docs-only/evaluation.rst +++ /dev/null @@ -1,5 +0,0 @@ -Evaluation - evaluation.py -=========================== - -.. .. automodule:: pyjedai.evaluation -.. :members: \ No newline at end of file diff --git a/docs-only/intersphinx.md b/docs-only/intersphinx.md deleted file mode 100644 index 2ae6592..0000000 --- a/docs-only/intersphinx.md +++ /dev/null @@ -1,28 +0,0 @@ -# Intersphinx: Cross-reference other projects - -Behind the built-in Sphinx extension `intersphinx` is a powerful tool to reference sections in other Sphinx and Jupyter Book documentation projects. - -You can configure mappings to external Sphinx projects in your Jupyter Book configuration, the `_config.yml` file. In this example project, we have configured `ebp` to reference `https://executablebooks.org/en/latest/`. In the following code examples, we refer to the configured `ebp` mapping and link directly to a section called `tools`. - -```{tab} MyST (Markdown) - -```{example} -We can link to pages in other documentation projects. -This is a link to the -[Executable Book project's list of tools they build](ebp:tools) -``` - - -```{tab} reStructuredText - -```{example} - -```{eval-rst} -We can link to pages in other documentation projects. -This is a link to the -:doc:`Executable Book project's list of tools they build ` -``` - -```{note} -In the above `reStructuredText` example, we use `{eval-rst}` to write reST inside a `.md` file (i.e. the one you are reading now). You only need to use this directive if you are writing reST code in a `.md` file. -``` diff --git a/docs-only/matching.rst b/docs-only/matching.rst deleted file mode 100644 index 53e7600..0000000 --- a/docs-only/matching.rst +++ /dev/null @@ -1,5 +0,0 @@ -Matching - matching.py -========================== - -.. .. automodule:: pyjedai.matching -.. :members: \ No newline at end of file diff --git a/docs-only/similarity_joins.rst b/docs-only/similarity_joins.rst deleted file mode 100644 index 9b1b570..0000000 --- a/docs-only/similarity_joins.rst +++ /dev/null @@ -1,6 +0,0 @@ -Similarity Joins - similarity_joins.py -====================================== - -.. .. autosummary:: -.. :toctree: _autosummary -.. :recursive: \ No newline at end of file diff --git a/docs-only/utils.rst b/docs-only/utils.rst deleted file mode 100644 index 3d9b2c0..0000000 --- a/docs-only/utils.rst +++ /dev/null @@ -1,5 +0,0 @@ -Utilities - utils.py -====================== - -.. .. automodule:: pyjedai.utils -.. :members: \ No newline at end of file diff --git a/documentation/3DERslide.png b/docs/img/3DERslide.png similarity index 100% rename from documentation/3DERslide.png rename to docs/img/3DERslide.png diff --git a/documentation/demo-architecture.png b/docs/img/demo-architecture.png similarity index 100% rename from documentation/demo-architecture.png rename to docs/img/demo-architecture.png diff --git a/documentation/logo_only.png b/docs/img/logo_only.png similarity index 100% rename from documentation/logo_only.png rename to docs/img/logo_only.png diff --git a/documentation/pLibTool.PNG b/docs/img/pLibTool.PNG similarity index 100% rename from documentation/pLibTool.PNG rename to docs/img/pLibTool.PNG diff --git a/documentation/pyJedAI-768x772.jpg b/docs/img/pyJedAI-768x772.jpg similarity index 100% rename from documentation/pyJedAI-768x772.jpg rename to docs/img/pyJedAI-768x772.jpg diff --git a/documentation/pyjedai-logo-lin.png b/docs/img/pyjedai-logo-lin.png similarity index 100% rename from documentation/pyjedai-logo-lin.png rename to docs/img/pyjedai-logo-lin.png diff --git a/documentation/pyjedai-ppt.png b/docs/img/pyjedai-ppt.png similarity index 100% rename from documentation/pyjedai-ppt.png rename to docs/img/pyjedai-ppt.png diff --git a/documentation/pyjedai.logo.drawio.png b/docs/img/pyjedai.logo.drawio.png similarity index 100% rename from documentation/pyjedai.logo.drawio.png rename to docs/img/pyjedai.logo.drawio.png diff --git a/documentation/pyjedai.logo.drawio.sm.png b/docs/img/pyjedai.logo.drawio.sm.png similarity index 100% rename from documentation/pyjedai.logo.drawio.sm.png rename to docs/img/pyjedai.logo.drawio.sm.png diff --git a/documentation/reading-process.jpg b/docs/img/reading-process.jpg similarity index 100% rename from documentation/reading-process.jpg rename to docs/img/reading-process.jpg diff --git a/documentation/reading-process.png b/docs/img/reading-process.png similarity index 100% rename from documentation/reading-process.png rename to docs/img/reading-process.png diff --git a/documentation/workflow-example.png b/docs/img/workflow-example.png similarity index 100% rename from documentation/workflow-example.png rename to docs/img/workflow-example.png diff --git a/documentation/workflow1-cora.png b/docs/img/workflow1-cora.png similarity index 100% rename from documentation/workflow1-cora.png rename to docs/img/workflow1-cora.png diff --git a/documentation/workflow1.png b/docs/img/workflow1.png similarity index 100% rename from documentation/workflow1.png rename to docs/img/workflow1.png diff --git a/documentation/workflow2-cora.png b/docs/img/workflow2-cora.png similarity index 100% rename from documentation/workflow2-cora.png rename to docs/img/workflow2-cora.png diff --git a/documentation/workflow2.png b/docs/img/workflow2.png similarity index 100% rename from documentation/workflow2.png rename to docs/img/workflow2.png diff --git a/documentation/workflow3.png b/docs/img/workflow3.png similarity index 100% rename from documentation/workflow3.png rename to docs/img/workflow3.png diff --git a/index.md b/index.md deleted file mode 100644 index 56204aa..0000000 --- a/index.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -layout: home ---- - - -
-
-pyJedAI -
-

-
-An open-source library that leverages Python’s data science ecosystem to build
powerful end-to-end Entity Resolution workflows. -
-
- ---- - -
- -[![Tests](https://github.com/Nikoletos-K/pyJedAI/actions/workflows/tests.yml/badge.svg?branch=main)](https://github.com/Nikoletos-K/pyJedAI/actions/workflows/tests.yml) -[![Linux](https://svgshare.com/i/Zhy.svg)](https://svgshare.com/i/Zhy.svg) -[![macOS](https://svgshare.com/i/ZjP.svg)](https://svgshare.com/i/ZjP.svg) -[![Windows](https://svgshare.com/i/ZhY.svg)](https://svgshare.com/i/ZhY.svg) -[![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/) - -# Overview - -pyJedAI is a python framework, aiming to offer experts and novice users, robust and fast solutions for multiple types of Entity Resolution problems. It is builded using state-of-the-art python frameworks. pyJedAI constitutes the sole open-source Link Discovery tool that is capable of exploiting the latest breakthroughs in Deep Learning and NLP techniques, which are publicly available through the Python data science ecosystem. This applies to both blocking and matching, thus ensuring high time efficiency, high scalability as well as high effectiveness, without requiring any labelled instances from the user. - -### Key-Features - -- Input data-type independent. Both structured and semi-structured data can be processed. -- Various implemented algorithms. -- Easy-to-use. -- Utilizes some of the famous and cutting-edge machine learning packages. -- Offers supervised and un-supervised ML techniques. - -__Open demos are available in:__ - -
- - -         - - - -
- -__Google Colab Hands-on demo:__ - -
- - - -
- -# Install - -Install the latest version of pyjedai __[requires python >= 3.7]__: -``` -pip install pyjedai -``` - -More on [PyPI](pypi.org/project/pyjedai/). - -__Find last release source code in [GitHub](https://github.com/AI-team-UoA/pyJedAI).__ - - -### Tutorials - -| Tutorial | Notebook | -|---|:-:| -| Clean-Clean Entity Resolution.| [CleanCleanER.ipynb](https://github.com/Nikoletos-K/pyJedAI/blob/main/tutorials/CleanCleanER.ipynb) | -| Dirty Entity Resolution. | [DirtyER.ipynb](https://github.com/Nikoletos-K/pyJedAI/blob/main/tutorials/DirtyER.ipynb)| -| Fine-Tuning using Optuna. | [Optuna.ipynb](https://github.com/Nikoletos-K/pyJedAI/blob/main/tutorials/Optuna.ipynb) | -| User-Friendly Approach. WorkFlow module. | [WorkFlow.ipynb](https://github.com/Nikoletos-K/pyJedAI/blob/main/tutorials/WorkFlow.ipynb) | -| Raw data to pandas DataFrame. | [Readers.ipynb](https://github.com/Nikoletos-K/pyJedAI/blob/main/tutorials/Readers.ipynb) | - -# Dependencies - -
-    -    -     -
- - -

-     -     -     - -
- -
- -See the full list of dependencies and all versions used, in this [file](https://github.com/Nikoletos-K/pyJedAI/blob/main/requirements.txt). - -# Bugs, Discussions & News - -[GitHub Discussions](https://github.com/Nikoletos-K/pyJedAI/discussions) is the discussion forum for general questions and discussions and our recommended starting point. Please report any bugs that you find [here](https://github.com/Nikoletos-K/pyJedAI/issues). - -# Team & Authors - -pyJedAI - -- [Konstantinos Nikoletos](https://nikoletos-k.github.io) -- [George Papadakis](https://gpapadis.wordpress.com) -- [Manolis Koubarakis](https://cgi.di.uoa.gr/~koubarak/) - -Research and development is made under the supervision of Pr. Manolis Koubarakis. This is a research project by the [AI-Team](https://ai.di.uoa.gr) of the Department of Informatics and Telecommunications at the University of Athens. - -# License - -Released under the Apache-2.0 license [(see LICENSE.txt)](https://github.com/Nikoletos-K/pyJedAI/blob/main/LICENSE). - -Copyright © 2022 AI-Team, University of Athens - -
-
-
- - -         - - - -

- This project is being funded in the context of STELAR, an HORIZON-Europe project.
-
- diff --git a/pyproject.toml b/pyproject.toml index 8510793..2d49299 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "pyjedai" -version = "0.0.6" +dynamic = ['version'] description = "An open-source library that builds powerful end-to-end Entity Resolution workflows." readme = "README.md" authors = [ @@ -63,6 +63,9 @@ dependencies = [ "py-stringmatching >= 0.4" ] +[tool.hatch.version] +path = 'src/pyjedai/_version.py' + [project.optional-dependencies] dev = ["pip-tools", "pytest"]