staging_install.py

#!/usr/bin/env python3
#
# Copyright 2020 OpenCircuitDesign
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
staging_install.py [options...]

This file copies the staging area created by foundry_install.py
into the target directory area, changing paths to match the target,
and creating symbolic links where requested and allowed.

Options:
  -staging <path>    Path to staging top level directory that the files
                     will be installed from.

  -target <path>     Final install path in system file system.

                     Normally, '$(prefix)/pdks/<unique pdk name>'.

                     If -local is not given, this will be the top level
                     directory location the files are installed too.

  -local <path>      Actual file system location to write the files too.
                     The result can then be packaged and distributed.

                     For usage with things like package managers and other
                     administrator installation tooling.  The resulting
                     files still need to be installed at '-target' on the
                     final system.

                     Think 'DESTDIR', see
                     https://www.gnu.org/prep/standards/html_node/DESTDIR.html

  -source <path>     Path to original source top level directory, if
                     link_from is "source".  This option may be called
                     multiple times if there are multiple sources.

  -variable <name>   Specify a variable name that is used for the
                     target path.  This variable name must be enforced
                     in setup scripts like .magicrc

Less common options:
  -link_from <type>  Make symbolic links to vendor files from target.

                     Types are: "none", "source", or a PDK name.

                     Default "none" (copy all files from source)

  -ef_format         Use efabless naming (libs.ref/techLEF),
                     otherwise use generic naming (libs.tech/lef)

If <target> is unspecified then <name> is used for the target.
"""

import re
import os
import sys
import glob
import stat
import shutil
import filecmp
import subprocess

# NOTE:  This version of copy_tree from distutils works like shutil.copytree()
# in Python 3.8 and up ONLY using "dirs_exist_ok=True".
from distutils.dir_util import copy_tree

def makeuserwritable(filepath):
    if os.path.exists(filepath):
        st = os.stat(filepath)
        os.chmod(filepath, st.st_mode | stat.S_IWUSR)

# Filter files to replace all strings matching "stagingdir" with "localdir" for
# every file in "tooldir".  If "tooldir" contains subdirectories, then recursively
# apply the replacement filter to all files in the subdirectories.  Do not follow
# symbolic links.

def filter_recursive(tooldir, stagingdir, localdir):
    # Add any non-ASCII file types here
    bintypes = ['.gds', '.gds2', '.gdsii', '.png', '.swp']

    # Also do substitutions on strings containing the stagingdir parent
    # directory (replace with the localdir parent directory).
    stagingparent = os.path.split(stagingdir)[0]
    localparent = os.path.split(localdir)[0]

    if not os.path.exists(tooldir):
        return 0
    elif os.path.islink(tooldir):
        return 0

    toolfiles = os.listdir(tooldir)
    total = 0

    for file in toolfiles:
        # Do not attempt to do text substitutions on a binary file!
        if os.path.splitext(file)[1] in bintypes:
            continue

        filepath = tooldir + '/' + file
        if os.path.islink(filepath):
            continue
        elif os.path.isdir(filepath):
            total += filter_recursive(filepath, stagingdir, localdir)
        else:
            with open(filepath, 'r') as ifile:
                try:
                    flines = ifile.read().splitlines()
                except UnicodeDecodeError:
                    print('Failure to read file ' + filepath + '; non-ASCII content.')
                    continue

            # Make sure this file is writable (as the original may not be)
            makeuserwritable(filepath)

            modified = False
            with open(filepath, 'w') as ofile:
                for line in flines:
                    newline = line.replace(stagingdir, localdir)
                    newline = newline.replace(stagingparent, localparent)
                    print(newline, file=ofile)
                    if newline != line:
                        modified = True

            if modified:
                total += 1
    return total

# To avoid problems with various library functions that copy hierarchical
# directory trees, remove all the files from the target that are going to
# be replaced by the contents of staging.  This avoids problems with
# symbolic links and such.

def remove_target(stagingdir, targetdir):

    slist = os.listdir(stagingdir)
    tlist = os.listdir(targetdir)

    for sfile in slist:
        if sfile in tlist:
            tpath = targetdir + '/' + sfile
            if os.path.islink(tpath):
                os.unlink(tpath)
            elif os.path.isdir(tpath):
                remove_target(stagingdir + '/' + sfile, targetdir + '/' + sfile)
            else:
                os.remove(tpath)

# Create a list of source files/directories from the contents of source.txt

def make_source_list(sources):
    sourcelist = []
    for source in sources:
        sourcelist.extend(glob.glob(source))
    return sourcelist

# Replace all files in list "libfiles" with symbolic links to files in
# "sourcelist", where the files are found to be the same.  If the entry
# in "libfiles" is a directory and the same directory is found in "sourcelist",
# then repeat recursively on the subdirectory.
#
# Because the installation may be distributed, there may be a difference
# between where the files to be linked to currently are (checklist)
# and where they will eventually be located (sourcelist).

def replace_with_symlinks(libfiles, sourcelist):
    # List of files that never get installed
    exclude = ['generate_magic.tcl', '.magicrc', 'sources.txt']
    total = 0
    for libfile in libfiles:
        if os.path.islink(libfile):
            continue
        else:
            try:
                sourcefile = next(item for item in sourcelist if os.path.split(item)[1] == os.path.split(libfile)[1])
            except:
                pass
            else:
                if os.path.isdir(libfile):
                    newlibfiles = glob.glob(libfile + '/*')
                    newsourcelist = glob.glob(sourcefile + '/*')
                    total += replace_with_symlinks(newlibfiles, newsourcelist)
                elif filecmp.cmp(libfile, sourcefile):
                    if not os.path.split(libfile)[1] in exclude:
                        os.remove(libfile)
                        # Use absolute path for the source file
                        sourcepath = os.path.abspath(sourcefile)
                        os.symlink(sourcepath, libfile)
                        total += 1
    return total

# Similar to the routine above, replace files in "libdir" with symbolic
# links to the files in "srclibdir", where the files are found to be the
# same.  The difference from the routine above is that "srclibdir" is
# another installed PDK, and so the directory hierarchy is expected to
# match that of "libdir" exactly, so the process of finding matches is
# a bit more straightforward.
#
# Because the installation may be distributed, there may be a difference
# between where the files to be linked to currently are (checklibdir)
# and where they will eventually be located (srclibdir).

def replace_all_with_symlinks(libdir, srclibdir, checklibdir):
    total = 0
    try:
        libfiles = os.listdir(libdir)
    except FileNotFoundError:
        print('Cannot list directory ' + libdir)
        print('Called: replace_all_with_symlinks(' + libdir + ', ' + srclibdir + ', ' + checklibdir + ')')
        return total

    try:
        checkfiles = os.listdir(checklibdir)
    except FileNotFoundError:
        print('Cannot list check directory ' + checklibdir)
        print('Called: replace_all_with_symlinks(' + libdir + ', ' + srclibdir + ', ' + checklibdir + ')')
        return total

    for libfile in libfiles:
        if libfile in checkfiles:
            libpath = libdir + '/' + libfile
            checkpath = checklibdir + '/' + libfile
            srcpath = srclibdir + '/' + libfile

            if os.path.isdir(libpath):
                if os.path.isdir(checkpath):
                    total += replace_all_with_symlinks(libpath, srcpath, checkpath)
            else:
                try:
                    if filecmp.cmp(libpath, checkpath):
                        os.remove(libpath)
                        os.symlink(srcpath, libpath)
                        total += 1
                except FileNotFoundError:
                    print('Failed file compare with libpath=' + libpath + ', checkpath=' + checkpath)

    return total

#----------------------------------------------------------------
# This is the main entry point for the staging install script.
#----------------------------------------------------------------

if __name__ == '__main__':

    if len(sys.argv) == 1:
        print("No options given to staging_install.py.")
        print(__doc__)
        sys.exit(0)

    optionlist = []
    newopt = []

    stagingdir = None
    targetdir = None
    link_from = None
    localdir = None
    variable = None

    ef_format = False
    do_install = True

    # Break arguments into groups where the first word begins with "-".
    # All following words not beginning with "-" are appended to the
    # same list (optionlist).  Then each optionlist is processed.
    # Note that the first entry in optionlist has the '-' removed.

    for option in sys.argv[1:]:
        if option.find('-', 0) == 0:
            if newopt != []:
                optionlist.append(newopt)
                newopt = []
            newopt.append(option[1:])
        else:
            newopt.append(option)

    if newopt != []:
        optionlist.append(newopt)

    # Check for option "ef_format" or "std_format"
    for option in optionlist[:]:
        if option[0] == 'ef_naming' or option[0] == 'ef_names' or option[0] == 'ef_format':
            optionlist.remove(option)
            ef_format = True
        elif option[0] == 'std_naming' or option[0] == 'std_names' or option[0] == 'std_format':
            optionlist.remove(option)
            ef_format = False
        elif option[0] == 'uninstall':
            optionlist.remove(option)
            do_install = False

    # Check for options "link_from", "staging", "target", and "local"

    link_name = None
    for option in optionlist[:]:
        if option[0] == 'link_from':
            optionlist.remove(option)
            if option[1].lower() == 'none':
                link_from = None
            elif option[1].lower() == 'source':
                link_from = 'source'
            else:
                link_from = option[1]
                link_name = os.path.split(link_from)[1]
        elif option[0] == 'staging' or option[0] == 'source':
            optionlist.remove(option)
            stagingdir = option[1]
        elif option[0] == 'target':
            optionlist.remove(option)
            targetdir = option[1]
        elif option[0] == 'local':
            optionlist.remove(option)
            localdir = option[1]
        elif option[0] == 'variable':
            optionlist.remove(option)
            variable = option[1]

    # Error if no staging or dest specified
    if not stagingdir:
        print("No staging directory specified.  Exiting.")
        sys.exit(1)

    if not targetdir:
        print("No target directory specified.  Exiting.")
        sys.exit(1)

    # If localdir is not specified, then it is the same as the parent
    # of the target (local installation assumed)
    if not localdir:
        localdir = targetdir

    # Take the target PDK name from the target path last component
    pdkname = os.path.split(targetdir)[1]

    # If link source is a PDK name, if it has no path, then pull the
    # path from the target name.

    if link_from:
        if link_from != 'source':
            if link_from.find('/', 0) < 0:
                link_name = link_from
                link_from = os.path.split(localdir)[0] + '/' + link_name
        else:
            # If linking from source, convert the source path to an
            # absolute pathname.
            stagingdir = os.path.abspath(stagingdir)

        # If link_from is the same as localdir, then set link_from to None
        if link_from == localdir:
            link_from = None

    # checkdir is the DIST target directory for the PDK pointed
    # to by link_name.  Files must be found there before creating
    # symbolic links to the (not yet existing) final install location.

    if link_name:
        checkdir = os.path.split(targetdir)[0] + '/' + link_name
    else:
        checkdir = ''

    # Diagnostic
    if do_install:
        print("Installing in target directory " + targetdir)
    else:
        print("Uninstalling from target directory " + targetdir)
        print("(Method not yet implemented)")

    # Create the top-level directories

    os.makedirs(targetdir, exist_ok=True)
    os.makedirs(targetdir + '/libs.tech', exist_ok=True)
    os.makedirs(targetdir + '/libs.ref', exist_ok=True)
    if os.path.isdir(stagingdir + '/libs.priv'):
        os.makedirs(targetdir + '/libs.priv', exist_ok=True)
        has_priv = True
    else:
        has_priv = False

    # Path to magic techfile depends on ef_format

    if ef_format == True:
        mag_current = '/libs.tech/magic/current/'
    else:
        mag_current = '/libs.tech/magic/'

    # First install everything by direct copy.  Keep the staging files
    # as they will be used to reference the target area to know which
    # files need to be checked and/or modified.

    if not os.path.isdir(targetdir):
        try:
            os.makedirs(targetdir, exist_ok=True)
        except:
            print('Fatal error:  Cannot make target directory ' + targetdir + '!')
            exit(1)

    # Remove any files from the target directory that are going to be replaced
    print('Removing files from target')
    remove_target(stagingdir, targetdir)

    print('Copying staging files to target')
    # print('Diagnostic:  copy_tree ' + stagingdir + ' ' + targetdir)
    copy_tree(stagingdir, targetdir, preserve_symlinks=True)
    print('Done.')

    # Magic and qflow setup files have references to the staging area that have
    # been used by the vendor install;  these need to be changed to the target
    # directory.

    print('Changing local path references from ' + stagingdir + ' to ' + localdir)
    print('Part 1:  Tools')

    needcheck = ['ngspice']
    techdirs = ['/libs.tech/']
    if has_priv:
        techdirs.append('/libs.priv/')

    for techdir in techdirs:
        tools = os.listdir(targetdir + techdir)
        for tool in tools:
            tooldir = targetdir + techdir + tool

            # There are few enough tool setup files that they can just all be
            # filtered directly.  This code only looks in the directory 'tooldir'.
            # If there are files is subdirectories of 'tooldir' that require
            # substitution, then this code needs to be revisited.

            # Note that due to the low overhead of tool setup files, there is
            # no attempt to check for possible symlinks to link_from if link_from
            # is a base PDK.

            total = filter_recursive(tooldir, stagingdir, localdir)
            if total > 0:
                substr = 'substitutions' if total > 1 else 'substitution'
                print('      ' + tool + ' (' + str(total) + ' ' + substr + ')')

    # If "link_from" is another PDK, then check all files against the files in
    # the other PDK, and replace the file with a symbolic link if the file contents
    # match (Note:  This is done only for ngspice model files;  other tool files are
    # generally small and deemed unnecessary to make symbolic links).

    if link_from not in ['source', None]:
        thispdk = os.path.split(targetdir)[1]

        # Only create links for PDKs other than the one we are making links to.
        if thispdk != link_from:
            print('Replacing files with symbolic links to ' + link_from + ' where possible.')
            for techdir in techdirs:
                for tool in needcheck:
                    tooldir = targetdir + techdir + tool
                    srctooldir = link_from + techdir + tool
                    if checkdir != '':
                        checktooldir = checkdir + techdir + tool
                    else:
                        checktooldir = srctooldir
                    if os.path.exists(tooldir):
                        total = replace_all_with_symlinks(tooldir, srctooldir, checktooldir)
                        if total > 0:
                            symstr = 'symlinks' if total > 1 else 'symlink'
                            print('      ' + tool + ' (' + str(total) + ' ' + symstr + ')')

    # In .mag files in mag/ and maglef/, also need to change the staging
    # directory name to localdir.  If "-variable" is specified in the options,
    # the replace the staging path with the variable name, not localdir.

    if variable:
        localname = '$' + variable
    else:
        localname = localdir

    needcheck = ['mag', 'maglef']
    refdirs = ['/libs.ref/']
    if has_priv:
        refdirs.append('/libs.priv/')

    if ef_format:
        print('Part 2:  Formats')
        for refdir in refdirs:
            for filetype in needcheck:
                print('   ' + filetype)
                filedir = targetdir + refdir + filetype
                if os.path.isdir(filedir):
                    libraries = os.listdir(filedir)
                    for library in libraries:
                        libdir = filedir + '/' + library
                        total = filter_recursive(libdir, stagingdir, localname)
                        if total > 0:
                            substr = 'substitutions' if total > 1 else 'substitution'
                            print('      ' + library + ' (' + str(total) + ' ' + substr + ')')
    else:
        print('Part 2:  Libraries')
        for refdir in refdirs:
            libraries = os.listdir(targetdir + refdir)
            for library in libraries:
                print('   ' + library)
                for filetype in needcheck:
                    filedir = targetdir + refdir + library + '/' + filetype
                    total = filter_recursive(filedir, stagingdir, localname)
                    if total > 0:
                        substr = 'substitutions' if total > 1 else 'substitution'
                        print('      ' + filetype + ' (' + str(total) + ' ' + substr + ')')

    # If "link_from" is "source", then check all files against the source
    # directory, and replace the file with a symbolic link if the file
    # contents match.  The "foundry_install.py" script should have added a
    # file "sources.txt" with the name of the source directories for each
    # install directory.

    if link_from not in ['source', None]:
        print('Replacing files with symbolic links to source where possible.')
        for refdir in refdirs:
            if ef_format:
                filedirs = os.listdir(targetdir + refdir)
                for filedir in filedirs:
                    print('   ' + filedir)
                    dirpath = targetdir + refdir + filedir
                    if os.path.isdir(dirpath):
                        libraries = os.listdir(dirpath)
                        for library in libraries:
                            libdir = targetdir + refdir + filedir + '/' + library
                            libfiles = os.listdir(libdir)
                            if 'sources.txt' in libfiles:
                                libfiles = glob.glob(libdir + '/*')
                                libfiles.remove(libdir + '/sources.txt')
                                with open(libdir + '/sources.txt') as ifile:
                                    sources = ifile.read().splitlines()
                                sourcelist = make_source_list(sources)
                                total = replace_with_symlinks(libfiles, sourcelist)
                                if total > 0:
                                    symstr = 'symlinks' if total > 1 else 'symlink'
                                    print('      ' + library + ' (' + str(total) + ' ' + symstr + ')')
            else:
                libraries = os.listdir(targetdir + refdir)
                for library in libraries:
                    print('   ' + library)
                    filedirs = os.listdir(targetdir + refdir + library)
                    for filedir in filedirs:
                        libdir = targetdir + refdir + library + '/' + filedir
                        if os.path.isdir(libdir):
                            libfiles = os.listdir(libdir)
                            if 'sources.txt' in libfiles:
                                # List again, but with full paths.
                                libfiles = glob.glob(libdir + '/*')
                                libfiles.remove(libdir + '/sources.txt')
                                with open(libdir + '/sources.txt') as ifile:
                                    sources = ifile.read().splitlines()
                                sourcelist = make_source_list(sources)
                                total = replace_with_symlinks(libfiles, sourcelist)
                                if total > 0:
                                    symstr = 'symlinks' if total > 1 else 'symlink'
                                    print('      ' + filedir + ' (' + str(total) + ' ' + symstr + ')')

    # Otherwise, if "link_from" is another PDK, then check all files against
    # the files in the other PDK, and replace the file with a symbolic link
    # if the file contents match.

    elif link_from:
        thispdk = os.path.split(targetdir)[1]

        # Only create links for PDKs other than the one we are making links to.
        if thispdk != link_from:

            print('Replacing files with symbolic links to ' + link_from + ' where possible.')

            for refdir in refdirs:
                if ef_format:
                    filedirs = os.listdir(targetdir + refdir)
                    for filedir in filedirs:
                        print('   ' + filedir)
                        dirpath = targetdir + refdir + filedir
                        if os.path.isdir(dirpath):
                            libraries = os.listdir(dirpath)
                            for library in libraries:
                                libdir = targetdir + refdir + filedir + '/' + library
                                srclibdir = link_from + refdir + filedir + '/' + library
                                if checkdir != '':
                                    checklibdir = checkdir + refdir + filedir + '/' + library
                                else:
                                    checklibdir = srclibdir
                                if os.path.exists(libdir):
                                    total = replace_all_with_symlinks(libdir, srclibdir, checklibdir)
                                    if total > 0:
                                        symstr = 'symlinks' if total > 1 else 'symlink'
                                        print('      ' + library + ' (' + str(total) + ' ' + symstr + ')')
                else:
                    libraries = os.listdir(targetdir + refdir)
                    for library in libraries:
                        print('   ' + library)
                        filedirs = os.listdir(targetdir + refdir + library)
                        for filedir in filedirs:
                            libdir = targetdir + refdir + library + '/' + filedir
                            srclibdir = link_from + refdir + library + '/' + filedir
                            if checkdir != '':
                                checklibdir = checkdir + refdir + library + '/' + filedir
                            else:
                                checklibdir = srclibdir
                            if os.path.exists(libdir):
                                total = replace_all_with_symlinks(libdir, srclibdir, checklibdir)
                                if total > 0:
                                    symstr = 'symlinks' if total > 1 else 'symlink'
                                    print('      ' + filedir + ' (' + str(total) + ' ' + symstr + ')')

    # Remove temporary files:  Magic generation scripts, sources.txt
    # file, and magic extract files.

    print('Removing temporary files from destination.')

    for refdir in refdirs:
        if ef_format:
            filedirs = os.listdir(targetdir + refdir)
            for filedir in filedirs:
                if os.path.islink(filedir):
                    continue
                elif os.path.isdir(filedir):
                    libraries = os.listdir(targetdir + refdir + filedir)
                    for library in libraries:
                        libdir = targetdir + refdir + filedir + '/' + library
                        libfiles = os.listdir(libdir)
                        for libfile in libfiles:
                            filepath = libdir + '/' + libfile
                            if os.path.islink(filepath):
                                continue
                            elif libfile == 'sources.txt':
                                os.remove(filepath)
                            elif libfile == 'generate_magic.tcl':
                                os.remove(filepath)
                            elif os.path.splitext(libfile)[1] == '.ext':
                                os.remove(filepath)
                            elif os.path.splitext(libfile)[1] == '.swp':
                                os.remove(filepath)
        else:
            libraries = os.listdir(targetdir + refdir)
            for library in libraries:
                filedirs = os.listdir(targetdir + refdir + library)
                for filedir in filedirs:
                    filepath = targetdir + refdir + library + '/' + filedir
                    if os.path.islink(filepath):
                        continue
                    elif os.path.isdir(filepath):
                        libfiles = os.listdir(filepath)
                        for libfile in libfiles:
                            libfilepath = filepath + '/' + libfile
                            if os.path.islink(libfilepath):
                                continue
                            elif libfile == 'sources.txt':
                                os.remove(libfilepath)
                            elif libfile == 'generate_magic.tcl':
                                os.remove(libfilepath)
                            elif os.path.splitext(libfile)[1] == '.ext':
                                os.remove(libfilepath)

    print('Done with PDK migration.')
    sys.exit(0)