From ac4d7dd90f1240d43dcc978fd59dee782aa21ef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Schl=C3=BCter?= Date: Fri, 2 Oct 2020 04:33:07 +0000 Subject: [PATCH 1/2] Remove redirect --- README.md | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 508161a7..d1c27826 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,114 @@ -## :warning: 301 Moved Permanently -Location: https://gitlab.com/kicad/libraries/kicad-library-utils +kicad-library-utils +=============== + +## / directory + +**check_lib_table.py**: Checks the validity of a library table against given libraries + +**download_pretty_libs.py**: Download or update KiCad version 4 footprint libraries + +## schlib directory + +**checklib.py**: Script for checking [KLC][] compliance of schematic symbol libraries. + +**schlib.py**: A Python module for parsing KiCad's schematic library file format. + +**test_schlib.sh**: A shell script used to validate the generation of files of the schlib module. + +**fix-pins.py**: A script created to help adapt existing library files to the [KiCad Library Convention][KLC] by testing some cases of x/y "wrong" pin positions and trying to fix them. The description of the cases are explained in the header of the script file. + +**move_part.py**: Script to move components between libraries. + +**autogen/**: Scripts for automatically generating schematic symbol libraries. + +## sch directory + +**sch.py**: A Python module for parsing KiCad's schematic file format. + +**test_sch.sh**: A shell script used to validate the generation of files of the sch module. + +**add_part_number.py**: This script is used to add/edit part number fields in the schematic files. + +**update_footprints.py**: This script updates the footprint fields of `.sch` files using a `.csv` file as input. + +## pcb directory + +**check_kicad_mod.py**: Script for checking [KLC][] compliance of footprint files. + +**kicad_mod.py**: A Python module for loading, editing, and saving KiCad footprint files. + +**check_3d_coverage.py**: Script for checking which KiCad footprints in a `.pretty` library have 3D models. It also shows unused 3D model files. + +[KLC]: http://kicad-pcb.org/libraries/klc/ + +How to use +========== + +## Schematic Library Checker + + # first get into schlib directory + cd kicad-library-utils/schlib + + # run the script passing the files to be checked + ./checklib.py path_to_lib1 path_to_lib2 + + # to check a specific component you can use the -c flag + ./checklib.py -c component_name path_to_lib1 + + # run the following 'h'elp command to see other options + ./checklib.py -h + +## Adding Part Number (PN) to Schematic Files + + # first get into sch directory + cd kicad-library-utils/sch + + # use the following command to add empty "MPN" fields in the schematic files + ./add_part_number.py path_to_sch1 path_to_sch2 + + # use the following command to add/edit the PN field using the passed csv file + # the default behaviour is search for "Reference(s)" and "MPN" columns in the csv + # the BOM generated by bom_csv_grouped_by_value plugin can be used after + # manually add the MPN field in the Collated Components section (for example) + ./add_part_number.py --bom-csv=path_to_bom_csv path_to_sch_files/*.sch + + # run the following 'h'elp command to see other options + ./add_part_number.py -h + + +## Footprint Checker + + # first get into pcb directory + cd kicad-library-utils/pcb + + # run the script passing the files to be checked + ./check_kicad_mod.py path_to_fp1.kicad_mod path_to_fp2.kicad_mod + + # Add `-v`, `-vv`, or `-vvv` for extra verbose output. The most useful is `-vv`, which explains in details the violations. Ex: + ./check_kicad_mod.py path_to_fp1.kicad_mod path_to_fp2.kicad_mod -vv + + # run the following 'h'elp command to see other options + ./check_kicad_mod.py -h + + +## 3D Coverage Checker + + # first get into pcb directory + cd kicad-library-utils/pcb + + # run the script to check all footprints + ./check_3d_coverage.py + + # run the script to check only the specified .pretty folder + ./check_3d_coverage.py --prettty Housings_SOIC + + # run the following 'h'elp command to see other options + ./check_3d_coverage.py -h + + +Notice +====== + +The scripts use a different algorithm to generate files in relation to the KiCad saving action. That will result output files with more modified lines than expected, because the line generally are repositioned. However, the file still functional. + +Always check the generated files by opening them on KiCad. Additionally, if you are working over a git repository (if not, you should) you can commit your work before proceed with the scripts, this will put you safe of any trouble. Also, you would use git diff to give a look at the modifications. \ No newline at end of file From 27f43e4f09873a03944551bb0c4b9eb8e560bc39 Mon Sep 17 00:00:00 2001 From: Raphael Lehmann Date: Wed, 20 May 2020 08:34:19 +0200 Subject: [PATCH 2/2] Update and improve schlib/autogen script for STM32 --- schlib/autogen/stm32/.gitignore | 3 + schlib/autogen/stm32/README.md | 13 ++- schlib/autogen/stm32/datasheets.txt | 111 ------------------- schlib/autogen/stm32/stm32_generator.py | 43 +++---- schlib/autogen/stm32/stm32_get_datasheets.py | 70 ++++++++++++ 5 files changed, 105 insertions(+), 135 deletions(-) create mode 100644 schlib/autogen/stm32/.gitignore delete mode 100644 schlib/autogen/stm32/datasheets.txt create mode 100755 schlib/autogen/stm32/stm32_get_datasheets.py diff --git a/schlib/autogen/stm32/.gitignore b/schlib/autogen/stm32/.gitignore new file mode 100644 index 00000000..277681f4 --- /dev/null +++ b/schlib/autogen/stm32/.gitignore @@ -0,0 +1,3 @@ +stm32_datasheets/ +db/ +mcu/ diff --git a/schlib/autogen/stm32/README.md b/schlib/autogen/stm32/README.md index aa7ab2d3..87f2ba97 100644 --- a/schlib/autogen/stm32/README.md +++ b/schlib/autogen/stm32/README.md @@ -6,9 +6,9 @@ STM32 devices. ## Prerequisites * XML files, taken from STM32CubeMX install (from db/mcu folder). -* Datasheet PDFs, downloaded from ST's website. A current list as of - 2018-02-06 can be found in datasheets.txt. These PDFs can be downloaded all - at once with `wget -P datasheets -ci datasheets.txt`. +* Datasheet PDFs, downloaded from ST's website. The helper tool + `stm32_get_datasheets.py` will download all available STM32 datasheets into + `./stm32_datasheets/`. * [pdfminer](https://github.com/euske/pdfminer) tool. ## Running @@ -16,5 +16,8 @@ STM32 devices. If you have the correct XML files, just run `./stm32_generator.py xmldir pdfdir`, where `xmldir` is a directory containing all the necessary XML files, and `pdfdir` is a directory containing all the PDF -files. Make sure the XML files have the correct format as there is no error -checking. +files (e.g. `./stm32_datasheets/`). Make sure the XML files have the correct +format as there is no error checking. + +Mining data from the pdf filestakes time, expect about 1-2 minutes per +datasheet and CPU core. diff --git a/schlib/autogen/stm32/datasheets.txt b/schlib/autogen/stm32/datasheets.txt deleted file mode 100644 index ad0b72f3..00000000 --- a/schlib/autogen/stm32/datasheets.txt +++ /dev/null @@ -1,111 +0,0 @@ -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00161561.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00161566.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00191174.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00191185.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210831.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210833.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210837.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210843.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00212417.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00220364.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00237391.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00251732.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00253739.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00253742.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00263874.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00277537.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00034689.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00035129.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00037051.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00039193.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00039232.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00046749.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00048356.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00049732.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00058181.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00059126.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00071990.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00077036.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00078075.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00078689.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00081611.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00086815.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00088500.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00090183.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00090510.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00092070.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00093332.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00093333.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00094064.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00097745.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00098321.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00098745.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00100431.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00101621.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00102166.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00102435.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00104043.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00105814.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00105960.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00106442.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108217.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108218.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108219.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108661.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108832.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108833.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00109263.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00109264.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00110868.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00111457.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00112716.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00115237.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00115249.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00115350.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00116561.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00118585.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00133117.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00135027.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00140359.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00140762.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141036.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141132.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141133.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141136.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141306.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141386.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00149404.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00149412.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00150658.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00150671.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00152023.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00162467.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00166114.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00166116.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00172872.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00206508.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00206858.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00208574.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00213872.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00214043.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00219980.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00225424.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00254865.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257192.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257195.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257205.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257211.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00273119.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00282247.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00282249.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00284207.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00284211.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00330506.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00330507.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00340475.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00340549.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00340637.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00366448.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00366449.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00387108.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00388325.pdf diff --git a/schlib/autogen/stm32/stm32_generator.py b/schlib/autogen/stm32/stm32_generator.py index ceafb967..02fd89d5 100755 --- a/schlib/autogen/stm32/stm32_generator.py +++ b/schlib/autogen/stm32/stm32_generator.py @@ -6,6 +6,8 @@ import os import re import sys +import multiprocessing +from itertools import repeat from lxml import etree @@ -344,9 +346,8 @@ def create_symbol(self, gen): f"{{ram}}KB RAM, {freqstr}{voltstr}{self.io} GPIO, " f"{pkgstr}") keywords = f"{self.core} {self.family} {self.line}" - datasheet = "" if self.pdf is None else (f"http://www.st.com/" - f"st-web-ui/static/active/en/resource/technical/document/" - f"datasheet/{self.pdf}") + datasheet = "" if self.pdf is None else (f"https://www.st.com/" + f"resource/en/datasheet/{self.pdf}") # Make the symbol self.symbol = gen.addSymbol(self.name, dcm_options={ @@ -670,6 +671,13 @@ def draw_symbol(self): visibility=SymbolField.FieldVisibility.INVISIBLE) +def run_pdf2txt(pdffile, pdfdir): + pdffile = os.path.join(pdfdir, pdffile) + pdfparsedfile = pdffile + ".par" + if not os.path.isfile(pdfparsedfile) and pdffile.endswith(".pdf"): + logging.info(f"Converting: {pdffile}") + os.system("pdf2txt.py -o " + pdfparsedfile + " " + pdffile) + def main(): parser = argparse.ArgumentParser( description='Generator for STM32 microcontroller symbols') @@ -696,28 +704,25 @@ def main(): # Parse text from PDFs for _, _, filenames in os.walk(args.pdfdir): - for pdffile in filenames: - pdffile = os.path.join(args.pdfdir, pdffile) - pdfparsedfile = pdffile + ".par" - if not os.path.isfile(pdfparsedfile) and pdffile.endswith(".pdf"): - logging.info(f"Converting: {pdffile}") - os.system("pdf2txt.py -o " + pdfparsedfile + " " + pdffile) - break + filenames.sort() + with multiprocessing.Pool() as pool: + pool.starmap(run_pdf2txt, zip(filenames, repeat(args.pdfdir))) # Load devices from XML, sorted by family libraries = {} for _, _, filenames in os.walk(args.xmldir): filenames.sort() for xmlfile in filenames: - # Load information about the part(s) - mcu = Device(os.path.join(args.xmldir, xmlfile), args.pdfdir) - # If there isn't a SymbolGenerator for this family yet, make one - if mcu.family not in libraries: - libraries[mcu.family] = SymbolGenerator( - lib_name=f"MCU_ST_{mcu.family}") - # If the part has a datasheet PDF, make a symbol for it - if mcu.pdf is not None: - mcu.create_symbol(libraries[mcu.family]) + if xmlfile.startswith("STM"): + # Load information about the part(s) + mcu = Device(os.path.join(args.xmldir, xmlfile), args.pdfdir) + # If there isn't a SymbolGenerator for this family yet, make one + if mcu.family not in libraries: + libraries[mcu.family] = SymbolGenerator( + lib_name=f"MCU_ST_{mcu.family}") + # If the part has a datasheet PDF, make a symbol for it + if mcu.pdf is not None: + mcu.create_symbol(libraries[mcu.family]) break # Write libraries diff --git a/schlib/autogen/stm32/stm32_get_datasheets.py b/schlib/autogen/stm32/stm32_get_datasheets.py new file mode 100755 index 00000000..8def4169 --- /dev/null +++ b/schlib/autogen/stm32/stm32_get_datasheets.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import json +import urllib.request +import shutil +import os +import multiprocessing + + +class Document: + def __init__(self, url, filename): + self.url = url + self.filename = filename + +class DocumentManager: + ds_urls = [ + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-high-performance-mcus.cxst-rs-grid.html/SC2154.technical_literature.datasheet.json', + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-mainstream-mcus.cxst-rs-grid.html/SC2155.technical_literature.datasheet.json', + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-ultra-low-power-mcus.cxst-rs-grid.html/SC2157.technical_literature.datasheet.json', + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-wireless-mcus.cxst-rs-grid.html/SC2156.technical_literature.datasheet.json', + #'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-arm-cortex-mpus.cxst-rs-grid.html/SC2230.technical_literature.datasheet.json', + ] + hdr = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', + 'Accept-Encoding': 'none', + 'Accept-Language': 'en-US,en;q=0.8', + 'Connection': 'keep-alive'} + + ds_list = list() + + _data_document_dir = 'stm32_datasheets/' + + def update_ds_list(self): + tmp_ds_list = list() + for url in self.ds_urls: + with urllib.request.urlopen(urllib.request.Request(url, headers=self.hdr)) as url: + data = json.loads(url.read().decode()) + rows = data['rows'] + for row in rows: + tmp_ds_list.append(Document( + url='https://www.st.com' + row['localizedLinks']['en'], + filename=row['localizedLinks']['en'].split('/')[-1], + ) + ) + self.ds_list = tmp_ds_list + + def download_pdf(self, d): + os.makedirs(self._data_document_dir, exist_ok=True) + if not os.path.isfile(self._data_document_dir + d.filename): + print('Downloading file {} ...'.format(d.url)) + with urllib.request.urlopen(urllib.request.Request(d.url, headers=self.hdr)) as response,\ + open(self._data_document_dir +d.filename, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + else: + print('Document {} already exists.'.format(d.filename)) + + def download(self): + with multiprocessing.Pool(10) as pool: + pool.map(self.download_pdf, self.ds_list) + +if __name__ == "__main__": + manager = DocumentManager() + manager.update_ds_list() + print("Found {} datasheets:".format(len(manager.ds_list))) + for ds in manager.ds_list: + print(ds.filename) + print("Downloading...") + manager.download()