- new: -wait command line option to change wait time between requests

- bugfix: whitelist function keeps results matching criterias from a CSV
- bugfix: blacklist function removes results matching criterias from a CSV

Author: GomoR

Changes

@@ -1,5 +1,10 @@
 Revision history for Perl extension Metabrik::Client::Onyphe.
 
+1.06 Fri 20 Mar 09:30:03 CET 2020
+   - new: -wait command line option to change wait time between requests
+   - bugfix: whitelist function keeps results matching criterias from a CSV
+   - bugfix: blacklist function removes results matching criterias from a CSV
+
 1.05 Wed 18 Mar 08:53:52 CET 2020
    - update: API v2
    - new: topsite, vulnscan, datamd5, domain and hostname API v2

LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2018-2019, ONYPHE
+Copyright (c) 2018-2020, ONYPHE
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

README

@@ -98,7 +98,7 @@ EXAMPLES
 
 COPYRIGHT AND LICENSE
 
-   Copyright (c) 2018-2019, ONYPHE
+   Copyright (c) 2018-2020, ONYPHE
 
    You may distribute this module under the terms of The BSD 3-Clause License.
    See LICENSE file in the source distribution archive.

bin/onyphe

@@ -16,6 +16,7 @@ my %lopts = (
    page => '1',
    autoscroll => '0',
    apiurl => 'https://www.onyphe.io/api/v2',
+   wait => 2,
 );
 
 # Option parsing
@@ -34,6 +35,7 @@ GetOptions(
    "maxpage=i" => \$lopts{maxpage},
    "autoscroll=i" => \$lopts{autoscroll},
    "apiurl=s" => \$lopts{apiurl},
+   "wait=i" => \$lopts{wait},
 ) or usage();
 
 # Load API key from onyphe rc file
@@ -48,6 +50,7 @@ $cli->log->level($lopts{verbose});
 $cli->apikey($lopts{apikey});
 $cli->apiurl($lopts{apiurl});
 $cli->autoscroll($lopts{autoscroll} ? 1 : 0);
+$cli->wait($lopts{wait});
 
 # Launch query
 my $r;
@@ -109,6 +112,7 @@ Usage: onyphe [options] -search 'category:CATEGORY SEARCH'
                                            or remove (-) from output
    -encode <FIELD1,FIELD2,...FIELDn>       comma separated list of fields to Base64 encode
    -verbose <0|1|2|3>                      verbosity level (default: 0)
+   -wait <SECONDS>                         wait time in regards to rate limit (default: 2)
 
 EOF
 ;
@@ -194,7 +198,7 @@ __END__
 
 =head1 COPYRIGHT AND LICENSE
 
-Copyright (c) 2018-2019, ONYPHE
+Copyright (c) 2018-2020, ONYPHE
 
 You may distribute this module under the terms of The BSD 3-Clause License.
 See LICENSE file in the source distribution archive.

lib/Metabrik/Api/Onyphe.pm

@@ -92,7 +92,9 @@ sub api {
       my $code = $self->code;
       if ($code == 429) {
          $self->log->verbose("api: request limit reached, waiting before retry");
-         sleep($wait);
+         if (defined($wait) && $wait > 0) {
+            sleep($wait);
+         }
          goto RETRY;
       }
       elsif ($code == 200) {
@@ -272,7 +274,7 @@ Metabrik::Api::Onyphe - api::onyphe Brik
 
 =head1 COPYRIGHT AND LICENSE
 
-Copyright (c) 2014-2019, Patrice E<lt>GomoRE<gt> Auffret
+Copyright (c) 2014-2020, Patrice E<lt>GomoRE<gt> Auffret
 
 You may distribute this module under the terms of The BSD 3-Clause License.
 See LICENSE file in the source distribution archive.

lib/Metabrik/Client/Onyphe.pm

@@ -5,7 +5,7 @@ package Metabrik::Client::Onyphe;
 use strict;
 use warnings;
 
-our $VERSION = '1.04';
+our $VERSION = '1.06';
 
 use base qw(Metabrik);
 
@@ -19,12 +19,14 @@ sub brik_properties {
          apikey => [ qw(apikey) ],
          apiurl => [ qw(apiurl) ],
          autoscroll => [ qw(0|1) ],
+         wait => [ qw(seconds) ],
          _ao => [ qw(INTERNAL) ],
          _sb => [ qw(INTERNAL) ],
       },
       attributes_default => {
          apiurl => 'https://www.onyphe.io/api/v2',
          autoscroll => 0,
+         wait => 2,
       },
       commands => {
          user => [ ],
@@ -356,6 +358,9 @@ sub function_merge {
    return $return->();
 }
 
+#
+# Will remove everything from input stream on matches.
+#
 sub function_blacklist {
    my $self = shift;
    my ($r, $lookup) = @_;
@@ -393,20 +398,28 @@ sub function_blacklist {
    for my $page (@$r) {
       for my $this (@{$page->{results}}) {
          my $skip = 0;
+         # $field is the field to match against (example: domain):
          for my $field (@$fields) {
-            if (defined($self->_value($this, $field))) {
+            # Fetch the value from current result $this:
+            my $value = $self->_value($this, $field);
+            if (defined($value)) {
+               # Compare against all fields given in the CSV:
                for my $h (@$l) {
-                  if (exists($h->{$field})
-                  &&  $h->{$field} ne $self->_value($this, $field)) {
+                  if (exists($h->{$field}) && $h->{$field} eq $value) {
+                     #print "[DEBUG] skip field [$field] value [$value]\n";
                      $skip++;
                      last;
                   }
                }
             }
-            last if $skip == $fields_count;
+            # When all fields have matched, no need to compare with remaining
+            if ($skip == $fields_count) {
+               #print "[DEBUG] all fields matched [$skip]\n";
+               last;
+            }
          }
-         if ($skip != $fields_count) { # Not all fields have matched,
-                                       # we keep results.
+         if ($skip != $fields_count) { # Not all fields have matched, it is
+                                       # NOT blacklisted, we keep results
             push @new, $this;
          }
       }
@@ -415,6 +428,9 @@ sub function_blacklist {
    return $return->();
 }
 
+#
+# Will keep everything from input stream on matches.
+#
 sub function_whitelist {
    my $self = shift;
    my ($r, $lookup) = @_;
@@ -452,18 +468,28 @@ sub function_whitelist {
    for my $page (@$r) {
       for my $this (@{$page->{results}}) {
          my $skip = 0;
+         # $field is the field to match against (example: domain):
          for my $field (@$fields) {
-            if (defined($self->_value($this, $field))) {
+            # Fetch the value from current result $this:
+            my $value = $self->_value($this, $field);
+            if (defined($value)) {
+               # Compare against all fields given in the CSV:
                for my $h (@$l) {
-                  if (exists($h->{$field}) && $h->{$field} eq $self->_value($this, $field)) {
+                  if (exists($h->{$field}) && $h->{$field} eq $value) {
+                     #print "[DEBUG] skip field [$field] value [$value]\n";
                      $skip++;
                      last;
                   }
                }
             }
-            last if $skip == $fields_count;
+            # When all fields have matched, no need to compare with remaining
+            if ($skip == $fields_count) {
+               #print "[DEBUG] all fields matched [$skip]\n";
+               last;
+            }
          }
-         if ($skip != $fields_count) { # No all fields have matched, we keep results.
+         if ($skip == $fields_count) { # All fields have matched, it is
+                                       # whitelisted, we keep results.
             push @new, $this;
          }
       }
@@ -895,7 +921,7 @@ L<Metabrik>
 
 =head1 COPYRIGHT AND LICENSE
 
-Copyright (c) 2018-2019, ONYPHE
+Copyright (c) 2018-2020, ONYPHE
 
 You may distribute this module under the terms of The BSD 3-Clause License.
 See LICENSE file in the source distribution archive.