Simplify hack/verify-flags-underscore.py

hack/verify-flags-underscore.py

@@ -25,7 +25,6 @@ import argparse
 
 parser = argparse.ArgumentParser()
 parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*')
-parser.add_argument("-e", "--skip-exceptions", help="ignore hack/verify-flags/exceptions.txt and print all output", action="store_true")
 args = parser.parse_args()
 
 # Cargo culted from http://stackoverflow.com/questions/898669/how-can-i-detect-if-a-file-is-binary-non-text-in-python
@@ -81,77 +80,10 @@ def get_all_files(rootdir):
             all_files.append(pathname)
     return all_files
 
-def normalize_files(rootdir, files):
-    newfiles = []
-    a = ['Godeps', '_gopath', 'third_party', '.git', 'exceptions.txt', 'known-flags.txt']
-    for f in files:
-        if any(x in f for x in a):
-            continue
-        if f.endswith(".svg"):
-            continue
-        if f.endswith(".gliffy"):
-            continue
-        if f.endswith(".md"):
-            continue
-        if f.endswith(".yaml"):
-            continue
-        newfiles.append(f)
-    for i, f in enumerate(newfiles):
-        if not os.path.isabs(f):
-            newfiles[i] = os.path.join(rootdir, f)
-    return newfiles
-
-def line_has_bad_flag(line, flagre):
-    results  = flagre.findall(line)
-    for result in results:
-        if not "_" in result:
-            return False
-        # this should exclude many cases where jinja2 templates use kube flags
-        # as variables, except it uses _ for the variable name
-        if "{% set" + result + "= \"" in line:
-            return False
-        if "pillar[" + result + "]" in line:
-            return False
-        if "grains" + result in line:
-            return False
-         # something common in juju variables...
-        if "template_data[" + result + "]" in line:
-            return False
-        return True
-    return False
-
-def check_known_flags(rootdir):
-    pathname = os.path.join(rootdir, "hack/verify-flags/known-flags.txt")
-    f = open(pathname, 'r')
-    flags = set(f.read().splitlines())
-    f.close()
-
-    illegal_known_flags = set()
-    for flag in flags:
-        if len(flag) > 0:
-            if not "-" in flag:
-                illegal_known_flags.add(flag)
-
-    if len(illegal_known_flags) != 0:
-        print("All flags in hack/verify-flags/known-flags.txt should contain character -, found these flags without -")
-        l = list(illegal_known_flags)
-        l.sort()
-        print("%s" % "\n".join(l))
-        sys.exit(1)
-
-
-# The list of files might not be the whole repo. If someone only changed a
-# couple of files we don't want to run all of the golang files looking for
-# flags. Instead load the list of flags from hack/verify-flags/known-flags.txt
-# If running the golang files finds a new flag not in that file, return an
-# error and tell the user to add the flag to the flag list.
-def get_flags(rootdir, files):
-    # preload the 'known' flags
-    pathname = os.path.join(rootdir, "hack/verify-flags/known-flags.txt")
-    f = open(pathname, 'r')
-    flags = set(f.read().splitlines())
-    f.close()
-
+# Collects all the flags used in golang files and verifies the flags do
+# not contain underscore. If any flag needs to be excluded from this check,
+# need to add that flag in hack/verify-flags/excluded-flags.txt.
+def check_underscore_in_flags(rootdir, files):
     # preload the 'known' flags which don't follow the - standard
     pathname = os.path.join(rootdir, "hack/verify-flags/excluded-flags.txt")
     f = open(pathname, 'r')
@@ -165,7 +97,6 @@ def get_flags(rootdir, files):
                re.compile('.Duration[P]?\("([^"]*)",[^,]+,[^)]+\)'),
                re.compile('.StringSlice[P]?\("([^"]*)",[^,]+,[^)]+\)') ]
 
-    new_flags = set()
     new_excluded_flags = set()
     # walk all the files looking for any flags being declared
     for pathname in files:
@@ -182,10 +113,6 @@ def get_flags(rootdir, files):
                 continue
             if "_" in flag:
                 new_excluded_flags.add(flag)
-            if not "-" in flag:
-                continue
-            if flag not in flags:
-                new_flags.add(flag)
     if len(new_excluded_flags) != 0:
         print("Found a flag declared with an _ but which is not explicitly listed as a valid flag name in hack/verify-flags/excluded-flags.txt")
         print("Are you certain this flag should not have been declared with an - instead?")
@@ -193,79 +120,17 @@ def get_flags(rootdir, files):
         l.sort()
         print("%s" % "\n".join(l))
         sys.exit(1)
-    if len(new_flags) != 0:
-        print("Found flags with character - in golang files not in the list of known flags. Please add these to hack/verify-flags/known-flags.txt")
-        l = list(new_flags)
-        l.sort()
-        print("%s" % "\n".join(l))
-        sys.exit(1)
-    return list(flags)
-
-def flags_to_re(flags):
-    """turn the list of all flags we found into a regex find both - and _ versions"""
-    dashRE = re.compile('[-_]')
-    flagREs = []
-    for flag in flags:
-        # turn all flag names into regexs which will find both types
-        newre = dashRE.sub('[-_]', flag)
-        # only match if there is not a leading or trailing alphanumeric character
-        flagREs.append("[^\w${]" + newre + "[^\w]")
-    # turn that list of regex strings into a single large RE
-    flagRE = "|".join(flagREs)
-    flagRE = re.compile(flagRE)
-    return flagRE
-
-def load_exceptions(rootdir):
-    exceptions = set()
-    if args.skip_exceptions:
-        return exceptions
-    exception_filename = os.path.join(rootdir, "hack/verify-flags/exceptions.txt")
-    exception_file = open(exception_filename, 'r')
-    for exception in exception_file.read().splitlines():
-        out = exception.split(":", 1)
-        if len(out) != 2:
-            print("Invalid line in exceptions file: %s" % exception)
-            continue
-        filename = out[0]
-        line = out[1]
-        exceptions.add((filename, line))
-    return exceptions
 
 def main():
     rootdir = os.path.dirname(__file__) + "/../"
     rootdir = os.path.abspath(rootdir)
 
-    exceptions = load_exceptions(rootdir)
-
     if len(args.filenames) > 0:
         files = args.filenames
     else:
         files = get_all_files(rootdir)
-    files = normalize_files(rootdir, files)
-
-    check_known_flags(rootdir)
-
-    flags = get_flags(rootdir, files)
-    flagRE = flags_to_re(flags)
-
-    bad_lines = []
-    # walk all the file looking for any flag that was declared and now has an _
-    for pathname in files:
-        relname = os.path.relpath(pathname, rootdir)
-        f = open(pathname, 'r')
-        for line in f.read().splitlines():
-            if line_has_bad_flag(line, flagRE):
-                if (relname, line) not in exceptions:
-                    bad_lines.append((relname, line))
-        f.close()
 
-    if len(bad_lines) != 0:
-        if not args.skip_exceptions:
-            print("Found illegal 'flag' usage. If these are false negatives you should run `hack/verify-flags-underscore.py -e > hack/verify-flags/exceptions.txt` to update the list.")
-        bad_lines.sort()
-        for (relname, line) in bad_lines:
-            print("%s:%s" % (relname, line))
-        return 1
+    check_underscore_in_flags(rootdir, files)
 
 if __name__ == "__main__":
   sys.exit(main())