一、实验目的：

1.会建立索引
2.会获取词频
3.会使用向量空间模型计算相似度
二、实验内容：
获取新闻热词
索引文档
向量空间模型
三、实验设备及软件环境：
版本 Windows 10 家庭中文版
版本号 20H2
操作系统内部版本 19042.928
elasticsearch-6.5.1
kibana-6.5.1-windows-x86_64
logstash-7.12.0
四、实验过程及结果：
获取新闻热词

//代码清单2-14 获取新闻热词
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
public class GetTopTerms{
    public static void main(String[] args) throws IOException{
        Directory directory = FSDirectory.open(Paths.get("C:\\Users\\15749\\Desktop\\实验\\全文检索\\test"));
        IndexReader reader = DirectoryReader.open(directory);
        //因为只索引了一个文档，所以DocID为0
        //通过getTermVector获取content字段的词项
        Terms terms = reader.getTermVector(0,"content");
        //遍历词项
        TermsEnum termsEnum = terms.iterator();
        Map<String, Integer> map = new HashMap<String,Integer>();
        BytesRef thisTerm;
        while((thisTerm = termsEnum.next() )!=null){
            String termText = thisTerm.utf8ToString();//词项
            //通过totalTermFreq()方法获取词项频率
            map.put(termText, (int)termsEnum.totalTermFreq());
        }
        //按value排序
        List<Map.Entry<String, Integer>> sortedMap = new ArrayList<Map.Entry<String,Integer>>(map.entrySet());
        Collections.sort(sortedMap,new Comparator<Map.Entry<String, Integer>>(){
            public int compare(Map.Entry<String, Integer> o1,Map.Entry<String, Integer> o2){
                return (o2.getValue() - o1.getValue());
            }
        });
        getTopN(sortedMap,10);
    }
    //获取top-n
    public static void getTopN(List<Entry<String, Integer>> sortedMap,int N){
        for(int i = 0;i<N;i++){
            System.out.println(sortedMap.get(i).getKey()+":"+sortedMap.get(i).getValue());
        }
    }
}

索引文档

//代码清单2-13 索引文档
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.wltea.analyzer.lucene.IKAnalyzer;
public class IndexDocs{
    public static void main(String[] args) throws IOException{
        File newsfile = new File("C:\\Users\\15749\\Desktop\\实验\\全文检索\\test\\test.txt");
        String text1 = textToString(newsfile);
        Analyzer smcAnalyzer = new IKAnalyzer(true);
        IndexWriterConfig indexWriterConfig = new IndexWriterConfig(smcAnalyzer);
        indexWriterConfig.setOpenMode(OpenMode.CREATE);
        //索引的存储路径
        Directory directory = null;
        //索引的增删改由indexWriter创建
        IndexWriter indexWriter = null;
        directory = FSDirectory.open(Paths.get("C:\\Users\\15749\\Desktop\\实验\\全文检索\\test"));
        indexWriter = new IndexWriter(directory,indexWriterConfig);
        //新建FieldType,用于指定字段索引时的信息
        FieldType type = new FieldType();
        //索引时保存文档、词项频率、位置信息、偏移信息
        type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
        type.setStored(true);//原始字符串全部被保存在索引中
        type.setStoreTermVectors(true);//存储词项量
        type.setTokenized(true);//词条化
        Document doc1 = new Document();
        Field field1 = new Field("content",text1,type);
        doc1.add(field1);
        indexWriter.addDocument(doc1);
        indexWriter.close();
        directory.close();
    }
    public static String textToString(File file){
        StringBuilder result = new StringBuilder();
        try{
            //构造一个BufferedReader类来读取文件
            BufferedReader br = new BufferedReader(new FileReader(file));
            String str = null;
            //使用readLine方法，一次读一行
            while((str = br.readLine()) != null){
                result.append(System.lineSeparator()+str);
            }
            br.close();
        }catch(Exception e){
            e.printStackTrace();
        }
        return result.toString();
    }
}

向量空间模型

//代码清单1-2 向量空间模型
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
public class Vsm{
    public static double calCosSim(Map<String, Double> v1,Map<String, Double> v2){
        double sclar = 0.0,norm1 = 0.0,norm2 = 0.0,similarity = 0.0;
        Set<String> v1Keys = v1.keySet();
        Set<String> v2Keys = v2.keySet();
        Set<String> both = new HashSet<>();
        both.addAll(v1Keys);
        both.retainAll(v2Keys);
        System.out.println(both);
        for(String str1:both){
            sclar+=v1.get(str1)*v2.get(str1);
        }
        for(String str1: v1.keySet()){
            norm1+=Math.pow(v1.get(str1), 2);
        }
        for(String str2: v2.keySet()){
            norm2+=Math.pow(v2.get(str2), 2);
        }
        similarity = sclar/Math.sqrt(norm1*norm2);
        System.out.println("sclar:"+sclar);
        System.out.println("norm1:"+norm1);
        System.out.println("norm2:"+norm2);
        System.out.println("similarity:"+similarity);
        return similarity;
    }
    public static void main(String...args){
        Map<String,Double> m1 = new HashMap<>();
        m1.put("Hello", 1.0);
        m1.put("css", 2.0);
        m1.put("Lucene", 3.0);
        Map<String, Double> m2 = new HashMap<>();
        m2.put("Hello", 1.0);
        m2.put("word", 2.0);
        m2.put("Hadoop", 3.0);
        m2.put("java", 4.0);
        m2.put("html", 1.0);
        m2.put("css", 2.0);
        calCosSim(m1, m2);
    }
}

五、总结：
通过本次实验会建立索引，会获取词频，会使用向量空间模型计算相似度

实验成绩： 指导教师： 年 月 日