Allow unreferenced strings if they start with '_' (#1941)

As briefly discussed in #1937, this change will make it so that any string
identifier that starts with '_' can be unreferenced. Any anonymous strings
must still be referenced.

While testing this out I realized that an unreferenced string still had the
STRING_FLAG_FIXED_OFFSET set, which meant any unreferenced string would have a
fixed_offset of YR_UNDEFINED. To deal with this when we are reducing the rule
we unset STRING_FLAG_FIXED_OFFSET if the string is unreferenced.

libyara/parser.c

@@ -1071,13 +1071,26 @@ int yr_parser_reduce_rule_declaration_phase_2(
     // Only the heading fragment in a chain of strings (the one with
     // chained_to == NULL) must be referenced. All other fragments
     // are never marked as referenced.
+    //
+    // Any string identifier that starts with '_' can be unreferenced. Anonymous
+    // strings must always be referenced.
 
-    if (!STRING_IS_REFERENCED(string) && string->chained_to == NULL)
+    if (!STRING_IS_REFERENCED(string) && string->chained_to == NULL &&
+        (STRING_IS_ANONYMOUS(string) ||
+         (!STRING_IS_ANONYMOUS(string) && string->identifier[1] != '_')))
     {
       yr_compiler_set_error_extra_info(
           compiler, string->identifier) return ERROR_UNREFERENCED_STRING;
     }
 
+    // If a string is unreferenced we need to unset the FIXED_OFFSET flag so
+    // that it will match anywhere.
+    if (!STRING_IS_REFERENCED(string) && string->chained_to == NULL &&
+        STRING_IS_FIXED_OFFSET(string))
+    {
+      string->flags &= ~STRING_FLAGS_FIXED_OFFSET;
+    }
+
     strings_in_rule++;
 
     if (strings_in_rule > max_strings_per_rule)
@@ -1121,7 +1134,7 @@ int yr_parser_reduce_string_identifier(
   YR_STRING* string;
   YR_COMPILER* compiler = yyget_extra(yyscanner);
 
-  if (strcmp(identifier, "$") == 0)  // is an anonymous string ?
+  if (strcmp(identifier, "$") == 0)            // is an anonymous string ?
   {
     if (compiler->loop_for_of_var_index >= 0)  // inside a loop ?
     {

tests/test-rules.c

@@ -512,6 +512,16 @@ static void test_syntax()
       "rule test { strings: $a = \"a\" condition: for 3.14159 of them: ($) }",
       ERROR_INVALID_VALUE);
 
+  assert_error(
+      "rule test { strings: $a = \"a\" condition: true }",
+      ERROR_UNREFERENCED_STRING);
+
+  // String identifiers prefixed with '_' are allowed to be unreferenced.
+  // Any unreferenced string must be searched for anywhere.
+  assert_string_capture(
+      "rule test { strings: $a = \"AXS\" $_b = \"ERS\" condition: $a }",
+      "AXSERS", "ERS");
+
   YR_DEBUG_FPRINTF(1, stderr, "} // %s()\n", __FUNCTION__);
 }
 
@@ -523,6 +533,11 @@ static void test_anonymous_strings()
       "rule test { strings: $ = \"a\" $ = \"b\" condition: all of them }",
       "ab");
 
+  // Anonymous strings must be referenced.
+  assert_error(
+      "rule test { strings: $ = \"a\" condition: true }",
+      ERROR_UNREFERENCED_STRING);
+
   YR_DEBUG_FPRINTF(1, stderr, "} // %s()\n", __FUNCTION__);
 }
 

tests/util.h

@@ -183,6 +183,16 @@ void assert_hex_atoms(
     }                                                                   \
   } while (0);
 
+// Ensure that a particular string is found when scanning. This is useful for
+// making sure that unreferenced strings are searched for properly.
+// Specifically, making sure they have STRING_FLAGS_FIXED_OFFSET unset.
+#define assert_string_capture(rule, string, expected) do {              \
+    if (!capture_string(rule, string, expected)) {                      \
+      fprintf(stderr, "%s:%d: rule does not match\n",                   \
+              __FILE__, __LINE__);                                      \
+      exit(EXIT_FAILURE);                                               \
+    }                                                                   \
+  } while (0);
 
 #define assert_match_count(rule, string, count)                         \
   do {                                                                  \