데블스캠프2011/둘째날/Machine-Learning/NaiveBayesClassifier/김동준 - Revision history

Maintenance script: Repair batch-0005 pages from live compare

2026-03-27T00:44:55Z

Repair batch-0005 pages from live compare

imported>teledong at 14:52, 29 June 2011

2011-06-29T14:52:08Z

← Older revision		Revision as of 14:52, 29 June 2011
Line 123:		Line 123:
	}		}
	}		}
	return ~~(double)~~advantageResult ~~/ (double)(sectionTrain.length - 1)~~;		return advantageResult;
	}		}
	// 해당 File 변수에 대한 Index Section 과의 매치율을 보여주는 함수. 맞은 것과 틀린것, 그리고 그 것에 대한 판단 확률을 반환한다.		// 해당 File 변수에 대한 Index Section 과의 매치율을 보여주는 함수. 맞은 것과 틀린것, 그리고 그 것에 대한 판단 확률을 반환한다.

imported>teledong at 14:28, 29 June 2011

2011-06-29T14:28:17Z

← Older revision		Revision as of 14:28, 29 June 2011
Line 112:		Line 112:
	return Math.log(((double)sectionTrain[index].getSectionWordNumber(word) / sectionTrain[index].getSectionWordsNumber()) / ((double)ArticleAdvantage(index, word) / notInSectionWordTotalSum));		return Math.log(((double)sectionTrain[index].getSectionWordNumber(word) / sectionTrain[index].getSectionWordsNumber()) / ((double)ArticleAdvantage(index, word) / notInSectionWordTotalSum));
	}		}
	// 특정 단어에 대한 Advantage 부과함수. 해당 Index Section 에만 존재하는 단어일때 빈도에 따른 가산점을 부여한다. 가산은 해당단어수 / Section ~~전체단어~~ * 50 이다.		// 특정 단어에 대한 Advantage 부과함수. 해당 Index Section 에만 존재하는 단어일때 빈도에 따른 가산점을 부여한다. 가산은 해당단어수 / Section 전체기사수 * 50 이다.
	private double ArticleAdvantage(int index, String word) {		private double ArticleAdvantage(int index, String word) {
	double advantageResult = 0;		double advantageResult = 0;

imported>teledong at 14:27, 29 June 2011

2011-06-29T14:27:32Z

Show changes

imported>teledong at 05:24, 29 June 2011

2011-06-29T05:24:10Z

Show changes

imported>teledong at 17:10, 28 June 2011

2011-06-28T17:10:25Z

← Older revision		Revision as of 17:10, 28 June 2011
Line 108:		Line 108:
	}		}
	나름 88%의 적중률 ㅋㅋㅋㅋㅋ		나름 88%의 적중률 ㅋㅋㅋㅋㅋ
			P.S : 조만간 이 발적화 코드를 바꿀예정

imported>teledong at 13:49, 28 June 2011

2011-06-28T13:49:18Z

← Older revision		Revision as of 13:49, 28 June 2011
Line 107:		Line 107:
	}		}
	}		}
			나름 88%의 적중률 ㅋㅋㅋㅋㅋ

imported>teledong at 13:48, 28 June 2011

2011-06-28T13:48:47Z

Show changes

imported>teledong at 13:47, 28 June 2011

2011-06-28T13:47:53Z

New page

Describe 데블스캠프2011/둘째날/Machine-Learning/NaiveBayesClassifier/김동준 here

import java.io.*;
import java.util.*;

public class Native {
public static void main(String[] args) {
int economyNum = 0;
int politicNum = 0;
Map<String,Integer> economyWord = new HashMap<String,Integer>();
Map<String,Integer> politicWord = new HashMap<String,Integer>();
BufferedReader economyLearn;
BufferedReader politicLearn;
try {
economyLearn = new BufferedReader(new FileReader(new File("C:/Users/KDJ/Documents/Devils/svm_data.tar/package/train/economy/index.economy.db")));
politicLearn = new BufferedReader(new FileReader(new File("C:/Users/KDJ/Documents/Devils/svm_data.tar/package/train/politics/index.politics.db")));
String tmp = economyLearn.readLine();
while(tmp != null) {
String[] a = tmp.split(" ");
for(String wordTmp:a) {
if(wordTmp.length() == 1) {continue; }
if( economyWord.get(wordTmp) == null) {
economyNum++;
economyWord.put(wordTmp, 1);
}
else { economyWord.put(wordTmp, economyWord.get(wordTmp)+1); }
}
tmp = economyLearn.readLine();
}
tmp = politicLearn.readLine();
while(tmp != null) {
String[] a = tmp.split(" ");
for(String wordTmp:a) {
if(wordTmp.length() == 1) {continue; }
if (politicWord.get(wordTmp) == null ) {
politicNum++;
politicWord.put(wordTmp, 1);
}
else { politicWord.put(wordTmp, politicWord.get(wordTmp)+1); }
}
tmp = politicLearn.readLine();
}
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
try {
double ecoResultNum = 0;
double polResultNum = 0;
double posiNum = 0;
double negaNum = 0;
double reslt = Math.log((double)economyNum/politicNum);
BufferedReader targetDocument = new BufferedReader(new FileReader(new File("C:/Users/KDJ/Documents/Devils/svm_data.tar/package/test/economy/economy.txt")));
String tmp = targetDocument.readLine();
while(tmp != null) {
String[] str = tmp.split(" ");
for(String tmpStr:str) {
if(economyWord.get(tmpStr) == null) { ecoResultNum = 0; }
else { ecoResultNum = economyWord.get(tmpStr); }
if(politicWord.get(tmpStr) == null) { polResultNum = 0; }
else { polResultNum = politicWord.get(tmpStr); }
polResultNum+=1;
ecoResultNum+=1;
reslt += Math.log(ecoResultNum / polResultNum);
}
if(reslt < 0) { negaNum+= 1; }
else { posiNum += 1; }
reslt = 0;
tmp = targetDocument.readLine();
}
double accu = (posiNum / (posiNum+negaNum));
targetDocument.close();
posiNum = 0;
negaNum = 0;
polResultNum = 0;
ecoResultNum = 0;
reslt = Math.log((double)economyNum/politicNum);
targetDocument = new BufferedReader(new FileReader(new File("C:/Users/KDJ/Documents/Devils/svm_data.tar/package/test/politics/politics.txt")));
tmp = targetDocument.readLine();
while(tmp != null) {
String[] str = tmp.split(" ");
for(String tmpStr:str) {
if(economyWord.get(tmpStr) == null) { ecoResultNum = 0; }
else { ecoResultNum = economyWord.get(tmpStr); }
if(politicWord.get(tmpStr) == null) { polResultNum = 0; }
else { polResultNum = politicWord.get(tmpStr); }
polResultNum+=1;
ecoResultNum+=1;
reslt += Math.log(polResultNum / ecoResultNum);
}
if(reslt < 0) { negaNum+= 1; }
else { posiNum += 1; }
reslt = 0;
tmp = targetDocument.readLine();
}
double accu2 = ((double)posiNum / (posiNum+negaNum));
System.out.println((accu + accu2) / 2);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}

← Older revision		Revision as of 00:44, 27 March 2026
Line 34:		Line 34:
	while(sectionLearn.hasNextLine()) {		while(sectionLearn.hasNextLine()) {
	this.sectionArticleNum++;		this.sectionArticleNum++;
	String[] a = sectionLearn.nextLine().split("\\s+");		String[] a = sectionLearn.nextLine().split("\\s+");
	for(String wordTmp:a) {		for(String wordTmp:a) {
	if(isSkipData(wordTmp)) {continue;} // 1글자Data, 사이트, 블로그, 페이지 주소의 경우 연관성및 신뢰성이 떨어지므로 검색에서 제외		if(isSkipData(wordTmp)) {continue;} // 1글자Data, 사이트, 블로그, 페이지 주소의 경우 연관성및 신뢰성이 떨어지므로 검색에서 제외
Line 73:		Line 73:

	public class Analyzer {		public class Analyzer {
	private Trainer[] sectionTrain;		private Trainer[] sectionTrain;
	private int notInSectionArticleSum = 0;		private int notInSectionArticleSum = 0;
	private int notInSectionWordSum = 0;		private int notInSectionWordSum = 0;
Line 81:		Line 81:
	this.notInSectionArticleSum = 0;		this.notInSectionArticleSum = 0;
	for(int i = 0; i < sectionTrain.length; i++) {		for(int i = 0; i < sectionTrain.length; i++) {
	if(i != index) { notInSectionArticleSum += sectionTrain[i].getSectionArticleNumber(); }		if(i != index) { notInSectionArticleSum += sectionTrain[i].getSectionArticleNumber(); }
	}		}
	this.notInSectionWordTotalSum = 0;		this.notInSectionWordTotalSum = 0;
	for(int i = 0; i < sectionTrain.length; i++) {		for(int i = 0; i < sectionTrain.length; i++) {
	if(i != index) { notInSectionWordTotalSum += sectionTrain[i].getSectionWordsNumber(); }		if(i != index) { notInSectionWordTotalSum += sectionTrain[i].getSectionWordsNumber(); }
	}		}
	}		}
Line 92:		Line 92:
	this.notInSectionWordSum = 0;		this.notInSectionWordSum = 0;
	for(int i = 0; i < sectionTrain.length; i++) {		for(int i = 0; i < sectionTrain.length; i++) {
	if(i != index) { notInSectionWordSum += sectionTrain[i].getSectionWordNumber(word); }		if(i != index) { notInSectionWordSum += sectionTrain[i].getSectionWordNumber(word); }
	}		}
	}		}
Line 103:		Line 103:
	return reslt;		return reslt;
	}		}
	// Ln[p(S) / p(!S)] 값을 계산하는 함수. Index 에 대응한다.		// Ln[p(S) / p(!S)] 값을 계산하는 함수. Index 에 대응한다.
	private double getLnPsPns(int index) {		private double getLnPsPns(int index) {
	return Math.log((double)sectionTrain[index].getSectionArticleNumber() / notInSectionArticleSum);		return Math.log((double)sectionTrain[index].getSectionArticleNumber() / notInSectionArticleSum);
	}		}
	// Sigma Ln[p(Wi ^ S) / p(Wi ^ !S)] 값을 계산하는 함수. Index 에 대응한다. 단 특정 단어에 대한 Advantage 를 부과한다. (Advantage 함수 참조)		// Sigma Ln[p(Wi ^ S) / p(Wi ^ !S)] 값을 계산하는 함수. Index 에 대응한다. 단 특정 단어에 대한 Advantage 를 부과한다. (Advantage 함수 참조)
	private double getLnPwsPwns(int index, String word) {		private double getLnPwsPwns(int index, String word) {
	CalculateNotInSectionWord(index, word);		CalculateNotInSectionWord(index, word);
	return Math.log(((double)sectionTrain[index].getSectionWordNumber(word) / sectionTrain[index].getSectionWordsNumber()) / ((double)ArticleAdvantage(index, word) / notInSectionWordTotalSum));		return Math.log(((double)sectionTrain[index].getSectionWordNumber(word) / sectionTrain[index].getSectionWordsNumber()) / ((double)ArticleAdvantage(index, word) / notInSectionWordTotalSum));
	}		}
	// 특정 단어에 대한 Advantage 부과함수. 해당 Index Section 에만 존재하는 단어일때 빈도에 따른 가산점을 부여한다. 가산은 해당단어수 / Section 전체기사수 * 50 이다.		// 특정 단어에 대한 Advantage 부과함수. 해당 Index Section 에만 존재하는 단어일때 빈도에 따른 가산점을 부여한다. 가산은 해당단어수 / Section 전체기사수 * 50 이다.
Line 117:		Line 117:
	for(int i = 0; i < sectionTrain.length; i++) {		for(int i = 0; i < sectionTrain.length; i++) {
	if(i != index) {		if(i != index) {
	if(sectionTrain[i].getSectionWordNumber(word) == 1 && sectionTrain[index].getSectionWordNumber(word) > 1) {		if(sectionTrain[i].getSectionWordNumber(word) == 1 && sectionTrain[index].getSectionWordNumber(word) > 1) {
	advantageResult += (1 - ((double)sectionTrain[index].getSectionWordNumber(word) / sectionTrain[index].getSectionArticleNumber() * 50));		advantageResult += (1 - ((double)sectionTrain[index].getSectionWordNumber(word) / sectionTrain[index].getSectionArticleNumber() * 50));
	}		}
	else { advantageResult += sectionTrain[i].getSectionWordNumber(word); }		else { advantageResult += sectionTrain[i].getSectionWordNumber(word); }
	}		}
	}		}
Line 143:		Line 143:
	}		}
	// 생성자. File 갯수에 따라 지정을 해준다.		// 생성자. File 갯수에 따라 지정을 해준다.
	public Analyzer(File[] dataList) {		public Analyzer(File[] dataList) {
	this.sectionTrain = new Trainer[dataList.length];		this.sectionTrain = new Trainer[dataList.length];
	for(int i = 0; i < sectionTrain.length; i++) {		for(int i = 0; i < sectionTrain.length; i++) {
	sectionTrain[i] = new Trainer(dataList[i]);		sectionTrain[i] = new Trainer(dataList[i]);
	sectionTrain[i].TrainData();		sectionTrain[i].TrainData();
	}		}
	}		}
Line 157:		Line 157:

	public class Runner {		public class Runner {
	public static void main(String[] args) {		public static void main(String[] args) {
	File[] dbList =		File[] dbList =
	{		{
	new File("svm_data.tar/package/train/economy/index.economy.db"),		new File("svm_data.tar/package/train/economy/index.economy.db"),
Line 173:		Line 173:
	위의 주석처럼 필요없는 (http, //, blog, yahoo, empas, tistory 같은) 단어를 제외하고 작성할 수 있게 수정했습니다.		위의 주석처럼 필요없는 (http, //, blog, yahoo, empas, tistory 같은) 단어를 제외하고 작성할 수 있게 수정했습니다.
	각 단어중 특별한 단어에 Advantage 를 부과했는데, 단어가 해당 Section 에 Unique 하고 그 빈도가 클수록 Advantage를 크게 부과했습니다. (하지만 이도 분석한 Section에 상대적입니다.)		각 단어중 특별한 단어에 Advantage 를 부과했는데, 단어가 해당 Section 에 Unique 하고 그 빈도가 클수록 Advantage를 크게 부과했습니다. (하지만 이도 분석한 Section에 상대적입니다.)
	[[그래서 그런지 결과엔 0.5% 차이밖에 없더군요..]]		그래서 그런지 결과엔 0.5% 차이밖에 없더군요..
	이 결과를 볼 수 있었으면 좋겠네요 ^^;;		이 결과를 볼 수 있었으면 좋겠네요 ^^;;