preprocessor/getRawData.py at master · twerating/preprocessor · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
from gensim.models import Word2Vec
from preprocess import preprocess
from genTokenList import form_token_list
import os
from genVector import *
from subprocess import call
from threading import Thread
import scipy.io as sio
import numpy as np
# import queue
from multiprocessing import Queue

def chunck(l, n):
	tweet_list = []
	for i in xrange(0, len(l), n):
		yield l[i:i+n]

train_folder = '../dataCollection/2000train'
test_folder = '../dataCollection/2000test'

def file2vec(filename, folder):
	tweets = preprocess(open(folder + '/' + filename).read().replace('\n', ' '))
	return tweets

def getRawTrain():
	token_list = import_token_list()
	trainMatrix = []
	trainLabel = []
	cnt = 0
	for filename in os.listdir(train_folder):
		if filename[0] == '.':
			continue
		cnt += 1
		print cnt
		rating = filename[0]
		tweets = file2vec(filename, train_folder)
		token_dict = form_token_dict(tweets)
		trainMatrix.append(gen_vector(token_dict, token_list))
		trainLabel.append([int(rating)])

	sio.savemat("rawData/RawTrain2000-300.mat", {'trainMatrix':trainMatrix, 'trainLabel':trainLabel})

def getRawTest():
	token_list = import_token_list()
	testMatrix = []
	testLabel = []
	cnt = 0
	for filename in os.listdir(test_folder):
		if filename[0] == '.':
			continue
		cnt += 1
		print cnt
		rating = filename[0]
		tweets = file2vec(filename, test_folder)
		token_dict = form_token_dict(tweets)
		testMatrix.append(gen_vector(token_dict, token_list))
		testLabel.append([int(rating)])
	sio.savemat("rawData/RawTest2000-300.mat", {'testMatrix':testMatrix, 'testLabel':testLabel})


def main():
	getRawTrain()
	getRawTest()

if __name__ == '__main__':
	main()