cutter

#!/usr/bin/env python3
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 colorcolumn=80
#     ______________________________________ ______________________
#     \                                     | (_)     (_)    (_)   \
#      `.  http://pub.cl.uzh.ch/purl/cutter |  __________________   }
#        `-.............................____|_(                  )_/
#

import sys
import argparse
import regex
import Cutter

assert sys.version_info >= (3,5)


def main(argv):
    parser = argparse.ArgumentParser(
        description='Tokenize with rule-based language models.',
        formatter_class=argparse.RawTextHelpFormatter,
        epilog='profiles:\n  {}'.format(
            '\n  '.join(sorted('{} ({})'.format(s, Cutter.profiles[s])
                for s in Cutter.profiles.keys()))))
    parser.add_argument(
        '-v', '--verbose',
        action='count', default=0, help='be more verbose')
    parser.add_argument(
        '-a', '--abbr',
        nargs='*',
        type=argparse.FileType('r'),
        help='use abbreviation list')
    parser.add_argument(
        '-i', '--init',
        nargs='*',
        type=argparse.FileType('r'),
        help='use list of sentence-initial words')
    parser.add_argument(
        '-r', '--rule',
        nargs='*',
        type=argparse.FileType('r'),
        help='use ruleset')
    parser.add_argument(
        '-t', '--text',
        nargs='?',
        type=argparse.FileType('r'), default=sys.stdin,
        help='read from a file (default: stdin)')
    parser.add_argument(
        '-w', '--whitespace',
        action='store_true',
        dest='wt',
        help='return whitespace tokens')
    parser.add_argument(
        '-E', '--no-empty',
        action='store_false',
        dest='et',
        help='suppress empty tokens')
    parser.add_argument(
        '-T', '--no-tags',
        action='store_false',
        dest='tt',
        help='suppress tokenizer tags')
    parser.add_argument(
        '-p', '--positions',
        action='store_true',
        dest='positions',
        help='indicate positions of token characters in input string')
    parser.add_argument(
        '-s', '--structure',
        action='store_true',
        dest='structure',
        help='indicate structure (level in tokenization tree)')
#    parser.add_argument(
#        '-c', '--cache',
#        help='load model from cache')
#    parser.add_argument(
#        '-u', '--update-cache',
#        help='write model to cache')
    parser.add_argument(
        '--version',
        action='store_true',
        help='print version and exit')
    parser.add_argument(
        'profile',
        nargs='?',
        help='')

    if not len(argv):
        parser.print_help()
        exit(1)

    args = parser.parse_args(args=argv)

    if args.version:
        print('Cutter {}'.format(Cutter.__version__))
        exit(0)

    cutter = Cutter.Cutter(verbosity=args.verbose)

    if args.profile:
        try:
            cutter.load_profile(args.profile)
        except Cutter.LanguageProfileUndefined:
            print('Profile {} not defined.'.format(args.profile))
            exit(2)
    else:
        if args.abbr is not None:
            for file in args.abbr:
                with file as file:
                    cutter.add_abbrs(file)
        if args.init is not None:
            for file in args.init:
                with file as file:
                    cutter.add_inits(file)
        if args.rule is not None:
            for file in args.rule:
                with file as file:
                    cutter.add_rules(file)
    cutter.compile()

    flags = (args.wt * Cutter.WHITESPACE_TOKENS + args.et * Cutter.EMPTY_TOKENS
             + args.tt * Cutter.TOKENIZATION_TAGS)
    formatstr = ''
    if args.positions:
        formatstr += '{3}\t{4}\t'
    if args.structure:
        formatstr += '{2}\t'
    if args.tt:
        formatstr += '{1}\t'
    formatstr += '{0}'
    for line in args.text:
        pos = 0
        tokens = cutter.cut(line, flags)
        for token in tokens:
            print(formatstr.format(*token))
            pos = token[4]
        if args.tt and args.et:
            print(formatstr.format('', '+NL', 0, pos, pos))
        else:
            print()


if __name__ == '__main__':
    main(sys.argv[1:])