Skip to content

Commit

Permalink
Feat: support to remove prefixes & suffixes from text input format
Browse files Browse the repository at this point in the history
  • Loading branch information
Loyalsoldier committed Aug 15, 2024
1 parent 76a5805 commit 00f229c
Showing 1 changed file with 24 additions and 5 deletions.
29 changes: 24 additions & 5 deletions plugin/plaintext/text_in.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ func newTextIn(action lib.Action, data json.RawMessage) (lib.InputConverter, err
InputDir string `json:"inputDir"`
Want []string `json:"wantedList"`
OnlyIPType lib.IPType `json:"onlyIPType"`

RemovePrefixesInLine []string `json:"removePrefixesInLine"`
RemoveSuffixesInLine []string `json:"removeSuffixesInLine"`
}

if len(data) > 0 {
Expand Down Expand Up @@ -68,6 +71,9 @@ func newTextIn(action lib.Action, data json.RawMessage) (lib.InputConverter, err
InputDir: tmp.InputDir,
Want: wantList,
OnlyIPType: tmp.OnlyIPType,

RemovePrefixesInLine: tmp.RemovePrefixesInLine,
RemoveSuffixesInLine: tmp.RemoveSuffixesInLine,
}, nil
}

Expand All @@ -80,6 +86,9 @@ type textIn struct {
InputDir string
Want map[string]bool
OnlyIPType lib.IPType

RemovePrefixesInLine []string
RemoveSuffixesInLine []string
}

func (t *textIn) GetType() string {
Expand Down Expand Up @@ -239,22 +248,32 @@ func (t *textIn) walkRemoteFile(url, name string, entries map[string]*lib.Entry)
func (t *textIn) scanFile(reader io.Reader, entry *lib.Entry) error {
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
line := scanner.Text()

line, _, _ = strings.Cut(line, "#")
line, _, _ = strings.Cut(line, "//")
line, _, _ = strings.Cut(line, "/*")
line = strings.TrimSpace(line)
if line == "" {
continue
}

line = strings.ToLower(line)
for _, prefix := range t.RemovePrefixesInLine {
line = strings.TrimSpace(strings.TrimPrefix(line, strings.ToLower(strings.TrimSpace(prefix))))
}
for _, suffix := range t.RemoveSuffixesInLine {
line = strings.TrimSpace(strings.TrimSuffix(line, strings.ToLower(strings.TrimSpace(suffix))))
}
line = strings.TrimSpace(line)
if line == "" {
continue
}

if err := entry.AddPrefix(line); err != nil {
return err
}
}

if err := scanner.Err(); err != nil {
return err
}
Expand Down

0 comments on commit 00f229c

Please sign in to comment.