CADeepLearning/ProcessData.py at master · kashpop/CADeepLearning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import json, os, nltk, pickle

word2VecFile = "/GW/D5data-1/kpopat/SnopesDeepLearning/resources/gloveWiki6B/word2vec.6B.300d.txt"
JSONFiles = "/GW/D5data-1/kpopat/SnopesDeepLearning/resources/Snopes+Web_json"
#JSONFiles = "/GW/D5data-1/kpopat/SnopesDeepLearning/resources/test"

dataPklPath = "/GW/D5data-1/kpopat/SnopesDeepLearning/resources/Snopes+Web_pickle"

missingWordFilePath = "/GW/D5data-1/kpopat/SnopesDeepLearning/Workspace/missing_word.txt"

vocab = {}


def prepareVocab():
	print("Loading Vocab")
	with open(word2VecFile) as f:
		f.readline()
		for i, line in enumerate(f):
			fields = line.strip().split(" ")
			vocab[fields[0]] = i
			#vectors.append(list(map(float, fields[1:])))
	print("Vocab Loaded!")


def prepare_dataset():
	print("Preparing data")

	missingWordFile = open(missingWordFilePath,"w")
	missingWords = set()

	numClaims = 0
	#iterate over all json file -- one json per claim
	for jsonFileName in os.listdir(JSONFiles):
		if not jsonFileName.endswith(".json"):
			continue
		else:
			#load the json file
			with open(os.path.join(JSONFiles, jsonFileName)) as jsonFile:
				jsonData = json.load(jsonFile)

			print("Processing: "+jsonData["Claim_ID"])

			if(jsonData["Credibility"] == "true"):
				cred_label = 0
			else:
				cred_label = 1

			docVectorCollection = []

			#for each search result page in json
			for searchPage in jsonData["Google Results"]:

				#for each search result document in the search page
				for searchResult in searchPage["results"]:

					#read the document text and convert it to small case
					docText = searchResult["doc_text"]

					#skip the empty documents and snopes web pages
					if docText == "" or searchResult["domain"] == "www.snopes.com":
						continue

					##ONLY FIRST 50,000 CHARACTERS -- IS IT OKAY?
					docText = docText[:50000].lower()
					docText = docText.lower()
					docVector = []

					#for each word in the document get the pretrained embedding vectors
					for sent in nltk.sent_tokenize(docText):
						for word in nltk.word_tokenize(sent):
							if word in vocab:
								docVector.append(vocab[word])
							else:
								missingWords.add(word)

					docVectorCollection.append(docVector)

			print("\tAdding {} documents".format(len(docVectorCollection)))

			filePath = os.path.join(dataPklPath, os.path.splitext(jsonFileName)[0]+".p");
			if(not os.path.isfile(filePath)):
				pickle.dump((docVectorCollection,cred_label), open(filePath, "wb" ))

			numClaims = numClaims + 1

	#dump missing words
	for missedWord in missingWords:
		missingWordFile.write(missedWord+"\n")
	missingWordFile.close()

	print("Data prepared!")
	print("Loaded claims:",numClaims)

prepareVocab()
prepare_dataset()