下载的数据是乱码，不知道你们是怎么处理的

[jxlijunhao]

下载的数据是乱码，不知道你们是怎么处理的

[jxlijunhao]

@prozhuchen 我重新编码成utf-8就ok了

import os
import codecs
import pdb

data_train_path = 'user_tag_query.10W.TRAIN'
data_test_path = 'user_tag_query.10W.TEST'

fobj_train = open(data_train_path,'r')
fobj_test = open(data_test_path,'r')
fwobj_train = codecs.open('user_tag_query.10W.TRAIN_encode', 'w', 'utf-8')
fwobj_test = codecs.open('user_tag_query.10W.TEST_encode', 'w', 'utf-8')

count = 1
for line in fobj_train:
print count
count += 1
line = line.strip().split('\t')
line = [ w.decode('gbk','ignore') for w in line]
s = '\t'.join(line)
fwobj_train.write(s + '\n')
fwobj_train.close()

count = 1
for line in fobj_test:
print count
count += 1
line = line.strip().split('\t')
line = [ w.decode('gbk','ignore') for w in line]
s = '\t'.join(line)
fwobj_test.write(s + '\n')
fwobj_test.close()