fixed top text and util to work

Python/toptext.py

@@ -2,32 +2,33 @@ from mrjob.job import MRJob
 import util
 import util
 import heapq
 import heapq
 import jellyfish
 import jellyfish
+import json
 
 
 CAPACITY = 100
 CAPACITY = 100
 
 
 class MRWordFreqCount(MRJob):
 class MRWordFreqCount(MRJob):
 
 
+	def mapper_init(self):
+		with open("var_map.json") as var:
+			self.varible_score = json.reads(var)
+		with open("func_map.json") as func:
+			self.function_score = json.reads(func)
+
     def mapper(self, _, line):
     def mapper(self, _, line):
         file1, file2, total = line.split(",")
         file1, file2, total = line.split(",")
-        file1name = "pyfile" + file1 + ".txt"
         file2name = "pyfile" + file2 + ".txt"
         file2name = "pyfile" + file2 + ".txt"
-        for i in range(int(file1) + 1, int(total)):
-            compare1 = "pyfile" + str(i) + ".txt"
-            params = util.funcsim(file1name, compare1)
-            text1 = params[4]
-            text2 = params[5]
-            simscore = jellyfish.jaro_distance(text1, text2)
-            yield file1name, simscore
-            yield compare1, simscore
-        if file1 != file2:
-            for i in range(int(file2) + 1, int(total)):
-                compare1 = "pyfile" + str(i) + ".txt"
-                params = util.funcsim(file1name, compare1)
-                text1 = params[4]
-                text2 = params[5]
-                simscore = jellyfish.jaro_distance(text1, text2) 
-                yield file1name, simscore
-                yield compare1, simscore
+        for file_number in {file1, file2}:
+        	filename = "pyfile" + file_number + ".txt"
+        	file_text = filename.read()
+        	for i in range(int(file1) + 1, int(total)):
+        		comparison_file = "pyfile" + i + ".txt"
+        		comparison_text = comparison_file.read()
+        		ts = jellyfish.jaro_winkler(filename_text,comparison_text)
+        		for text in [(filename, file_text), (comparison_file, comparison_text)]:
+        			vs = util.get_variable_score(self.varible_score, text[1])
+        			fs = util.get_function_score(self.function_score, text[1])
+        			yield text[0], ts + vs + fs
+            
                 
                 
     def combiner(self, word, counts):
     def combiner(self, word, counts):
         cs = list(counts)
         cs = list(counts)

util.py

 import re
 import re
 import jellyfish
 import jellyfish
 
 
+REG_V = "[\s]*([a-zA-Z_][\w, ]*)[\s]*=[^=]"
+REG_F = "def [a-zA-Z_][\w]*\("
+DEFAULT_RETURN = 1
+
 def funcsim(file1name,file2name):
 def funcsim(file1name,file2name):
     count = 0
     count = 0
     with open(file1name) as f1:
     with open(file1name) as f1:
@@ -12,6 +16,24 @@ def funcsim(file1name,file2name):
 
 
     return (name1, name2, params1, params2, text1, text2)
     return (name1, name2, params1, params2, text1, text2)
 
 
+def get_variable_score(d, text):
+    variables = re.findall(REG_V, text)
+    num = len(variables)
+    total = 0
+    for variable in variables:
+        total += d.get(variable, DEFAULT_RETURN)
+    return total / num
+
+def get_function_score(d, text):
+    functions = re.findall(REG_F, text)
+    num = len(functions)
+    total = 0
+    for functions in functions:
+        total += d.get(function, DEFAULT_RETURN)
+    return total / num
+
+
+
 
 
 '''
 '''
 Functions for C
 Functions for C