SpeedNumCube/sncprojects/lib/SNCCore/numberexport.cpp

#include "numberexport.h"
#include "SNCDataBase.h"
NumberExport::NumberExport(NumberListModel *model,QString fileNamePrefix,enum NumberExportType exportType)
{
    this->model = model;
    this->fileNamePrefix = fileNamePrefix;
    this->stop = false;
    this->exportType = exportType;
    this->batchSize = 100000;
    this->isCity = true;
}

NumberExport::~NumberExport()
{
    qDebug()<<QDateTime::currentDateTime().toString("yyyy-MM-dd HH:mm:ss")\
           <<"NumberExport Destroyed.";
}
void NumberExport::abort()
{
    this->stop = true;
}
void NumberExport::setBatchSize(int batchSize)
{
    this->batchSize = batchSize;
}
void NumberExport::setCityOrProvince(bool isCity)
{
    this->isCity = isCity;
}
void NumberExport::setGroupSate(bool isAll)
{
    this->isAll = isAll;
}
void NumberExport::setSegment(int len)
{
    this->len = len;
}
void NumberExport::run()
{
    QTime t;
    t.start();
    switch(exportType)
    {
        case ExportByAll:
            exportByAll();
            break;
        case ExportByBatch:
            exportByBatch();
            break;
        case ExportByCorp:
            exportByCorp();
            break;
        case ExportByRegion:
            exportByRegion();
            break;
        case ExportByDuplicate:
            exportByDuplicate();
            break;
        case ExportBySegment:
            exportBySegmentByLen(this->len);
            break;
        default:
            break;
    }
    qDebug()<<QDateTime::currentDateTime().toString("yyyy-MM-dd HH:mm:ss")\
           <<"Number Export Elapsed Time"<<t.elapsed();
}
void NumberExport::exportByAll() //export by all
{
    QFile file(fileNamePrefix+"_"+QString::number(model->getCount())+".txt");
    if(!file.open(QIODevice::WriteOnly))
        return ;
    QTextStream out(&file);
    int progress = 0;
    int groups = SNCCore::getBatchGroups(model->getCount(),batchSize);
    emit progressMaxUpdated(groups);
    //fix on 2012-8-27
//    int count = 0;
//    for(int i=0;i<model->getSegments().size();i++)
//    {
//        for(int j=0;j<model->getNumbers().at(i).size();j++)
//        {
//            count++;
//            QString num = model->getSegments().at(i) + model->getNumbers().at(i).at(j);
//            out << num + "\r\n";
//            //qDebug()<<count<<num;
//            if(count % batchSize == 0  || count == model->getCount())
//            {
//                progress++;
//                emit progressUpdated(progress);
//            }
//        }
//    }
//    emit progressUpdated(groups);
    model->setMapIndex();
    for(int i=0;i<model->getCount();i++)
    {
        if(i<model->getCount()-1)
            out<<model->getNumberEx(i)<<"\r\n";
        else
            out<<model->getNumberEx(i);
        if(((i+1)%batchSize==0 && (i+1)/batchSize>0)|| i==model->getCount()-1)
        {
            progress++;
            emit progressUpdated(progress);
        }
    }
    file.close();
    emit exportFinished();
}
void NumberExport::exportByBatch()//export by group
{
    int size = model->getCount();
    emit progressMaxUpdated(size);
    int progress = qMax(size/50,1);
//    int count = 0;
//    int batch = 0;
//    int itemsToFetch = 0;
//    QString num ;
//    for(int i=0;i<model->getSegments().size();i++)
//    {
//        for(int j=0;j<model->getNumbers().at(i).size();j++)
//        {
//            count++;
//            itemsToFetch++;
//            num += model->getSegments().at(i) + model->getNumbers().at(i).at(j) + "\r\n";
//            if(count % batchSize == 0  || count == model->getCount())
//            {
//                QFile file(fileNamePrefix+"_第"+QString::number(++batch)+"批_"+QString::number(itemsToFetch)+".txt");
//                if(file.open(QIODevice::WriteOnly))
//                {
//                    QTextStream out(&file);
//                    out<<num;
//                    num = "";
//                    itemsToFetch = 0;
//                    file.close();
//                }

//            }
//            if( count % progress ==0 || count == size)
//            {
//                emit progressUpdated(count);
//            }
//        }
//    }
//    emit progressUpdated(size);
    model->setMapIndex();
    int groups = SNCCore::getBatchGroups(size,this->batchSize);
    for(int i=0;i<groups;i++)
    {
        if(stop)
            break;
        int remainder = size - i*this->batchSize;
        int itemsToFetch = qMin(this->batchSize, remainder);
        QFile file(fileNamePrefix+"_第"+QString::number(i+1)+"批_"+QString::number(itemsToFetch)+".txt");
        if(!file.open(QIODevice::WriteOnly))
            return ;
        QTextStream out(&file);
        QString content ="";
        for(int j=i*this->batchSize;j<i*this->batchSize+itemsToFetch;j++)
        {
            if( (j+1) % progress ==0 || j==size-1)
            {
                emit progressUpdated(j+1);
            }
            //content+=model->getNumberEx(j)+"\r\n";
            //fix-1000多万的号码，分两批导出，每个包5650000个号码，导出到44%的时候报错
            //modify by xxw on 2016-5-23
            content =model->getNumberEx(j)+"\r\n";
            out << content;
        }
        //fix-1000多万的号码，分两批导出，每个包5650000个号码，导出到44%的时候报错
        //modify by xxw on 2016-5-23
        //out << content;
        file.close();
        if(!isAll)
        {
            emit progressUpdated(size);
            break;
        }
    }
    emit exportFinished();
}
//modify by xxw on 2016-3-17
//增加虚拟运营商
void NumberExport::exportByCorp()//export by corp
{
    int size = this->model->getSegments().size();
    emit progressMaxUpdated(size);
    QFile fileCMCC(fileNamePrefix+"_移动.txt");
    QFile fileCTCC(fileNamePrefix+"_电信.txt");
    QFile fileCUTC(fileNamePrefix+"_联通.txt");
    QFile fileXNTC(fileNamePrefix+"_虚拟.txt");
    fileCMCC.open(QIODevice::WriteOnly);
    fileCTCC.open(QIODevice::WriteOnly);
    fileCUTC.open(QIODevice::WriteOnly);
    fileXNTC.open(QIODevice::WriteOnly);
    QTextStream outCMCC(&fileCMCC);
    QTextStream outCTCC(&fileCTCC);
    QTextStream outCUTC(&fileCUTC);
    QTextStream outXNTC(&fileXNTC);
    int l,m,n,p;
    l=0;
    m=0;
    n=0;
    p=0;
    for(int i=0;i<size;i++)
    {
        if(stop)
            break;
        QString segment = this->model->getSegments().at(i);
        if(NumberReg::getCMCCReg().exactMatch(segment+"0000"))
        {
            for(int j=0;j<this->model->getNumbers().at(i).size();j++)
            {
                outCMCC << segment+this->model->getNumbers().at(i).at(j)<<"\r\n";
                l++;
            }
        }
        if(NumberReg::getCTCCReg().exactMatch(segment+"0000"))
        {
            for(int j=0;j<this->model->getNumbers().at(i).size();j++)
            {
                outCTCC << segment+this->model->getNumbers().at(i).at(j)<<"\r\n";
                m++;
            }
        }
        if(NumberReg::getCUTCReg().exactMatch(segment+"0000"))
        {
            for(int j=0;j<this->model->getNumbers().at(i).size();j++)
            {
                outCUTC << segment+this->model->getNumbers().at(i).at(j)<<"\r\n";
                n++;
            }
        }
        if(NumberReg::getXNTCReg().exactMatch(segment+"0000"))
        {
            for(int j=0;j<this->model->getNumbers().at(i).size();j++)
            {
                outXNTC << segment+this->model->getNumbers().at(i).at(j)<<"\r\n";
                p++;
            }
        }
        emit progressUpdated(i+1);
    }
    fileCMCC.close();
    fileCTCC.close();
    fileCUTC.close();
    fileXNTC.close();
    if(l==0)
        QFile::remove(fileNamePrefix+"_移动.txt");
    if(m==0)
        QFile::remove(fileNamePrefix+"_电信.txt");
    if(n==0)
        QFile::remove(fileNamePrefix+"_联通.txt");
    if(p==0)
        QFile::remove(fileNamePrefix+"_虚拟.txt");
    if(l>0)
        QFile::rename(fileNamePrefix+"_移动.txt",fileNamePrefix+"_移动_"+QString::number(l)+".txt");
    if(m>0)
        QFile::rename(fileNamePrefix+"_电信.txt",fileNamePrefix+"_电信_"+QString::number(m)+".txt");
    if(n>0)
        QFile::rename(fileNamePrefix+"_联通.txt",fileNamePrefix+"_联通_"+QString::number(n)+".txt");
    if(p>0)
        QFile::rename(fileNamePrefix+"_虚拟.txt",fileNamePrefix+"_虚拟_"+QString::number(p)+".txt");
    emit exportFinished();
    //deleteLater();
}
void NumberExport::exportBySegmentByLen(int len)//export by segemnt len
{
    NumberListModel *newModel = new NumberListModel();
    newModel->setNumberList(this->model->getSegments(),this->model->getNumbers());
    newModel->mergeSegment();
    int size = newModel->getSegments().size();
    emit progressMaxUpdated(size);
    int progress = qMax(size/50,1) ;
    int count = 0;
    QString seg;
    QString buffer;
    for(int i=0;i<size;i++)
    {
        QString segment = newModel->getSegments().at(i);

        for(int j=0;j<newModel->getNumbers().at(i).size();j++)
        {
            QString num = segment+newModel->getNumbers().at(i).at(j)+"\r\n";
            QString key = num.left(len);

            if(seg.isEmpty())
            {
                buffer = num;
                seg = key;
                count = 1;
            }
            else
            {
                if(key!=seg)
                {
                    QFile file(fileNamePrefix+"_"+seg+"_"+ QString::number(count)+ ".txt");
                    if(file.open(QIODevice::WriteOnly))
                    {
                        QTextStream out(&file);
                        out << buffer;
                        file.close();
                        count = 0;
                    }
                    seg = key;
                    count = 1;
                    buffer = num;
                }
                else
                {
                    buffer += num;
                    count++;
                }
            }

        }
        if( i % progress == 0 || i == size-1)
            emit progressUpdated(i+1);
    }
    if(!buffer.isEmpty())
    {
        QFile file(fileNamePrefix+"_"+seg+"_"+ QString::number(count)+ ".txt");
        if(file.open(QIODevice::WriteOnly))
        {
            QTextStream out(&file);
            out << buffer;
            file.close();
        }
    }
    newModel->clear();
    delete newModel;
    emit exportFinished();
}
void NumberExport::exportByDuplicate()//export by times
{
    QStringList segments;
    QList<QStringList> numbers;

    NumberListModel *newModel = new NumberListModel();
    newModel->setNumberList(this->model->getSegments(),this->model->getNumbers());
    newModel->mergeSegment();
    segments = newModel->getSegments();
    numbers = newModel->getNumbers();
    int size = segments.size();
    emit progressMaxUpdated(size);
    QMap<int,int> map;

    for(int i=0;i<segments.size();i++)
    {
        numbers[i].sort();        
        QMultiMap<int,QString> multimap;
        int same = 1;
        QString number=numbers.at(i).at(0);
        QString data = segments.at(i) + number + "\r\n";
        for(int j=1;j<numbers.at(i).size();j++)
        {
            if(numbers.at(i).at(j) == number)
            {
                same++;
            }
            else
            {
                if(map.contains(same))
                    map.insert(same,map.value(same)+1);
                else
                    map.insert(same,1);
                multimap.insert(same,data);
                same = 1;
                number = numbers.at(i).at(j);
                data = segments.at(i) + number + "\r\n";
            }
        }
        if(map.contains(same))
            map.insert(same,map.value(same)+1);
        else
            map.insert(same,1);
        multimap.insert(same,data);
        foreach(int key,multimap.uniqueKeys())
        {
            QStringList list = multimap.values(key);
            QString f = fileNamePrefix+"_出现"+QString::number(key)+"次"+".txt";
            QFile file(f);
            file.open(QIODevice::Append);
            QTextStream out(&file);
            QString buffer;
            for( int i=0;i<list.size();i++)
            {
                buffer += list.at(i);
            }
            out<<buffer;
            file.close();
        }
        emit progressUpdated(i+1);
    }

    foreach(int key,map.keys())
    {
        int count = map.value(key);
        QString tmpName = fileNamePrefix+"_出现"+QString::number(key)+"次.txt";
        QString finalName = fileNamePrefix+"_出现"+QString::number(key)+"次_"+QString::number(count)+".txt";
        QFile::rename(tmpName,finalName);
    }
    newModel->clear();
    delete newModel;
    emit exportFinished();

}

//mod by wzx
void NumberExport::exportByRegion() //export by city
{
    QStringList segments;
    QList<QStringList> numbers;

    NumberListModel *tmpModel = new NumberListModel();
    tmpModel->setNumberList(this->model->getSegments(), this->model->getNumbers());
    tmpModel->mergeSegment();

    segments = tmpModel->getSegments();
    numbers = tmpModel->getNumbers();

    emit progressMaxUpdated(segments.size()+2);
    emit progressUpdated(1);
    QMap<QString,QString> mapSegments ;
    QMap<QString,QString> mapCityPhone ;
    if(this->isCity)
       mapSegments = SNCDataBase::groupSegmentIntoCity(segments);
    else
       mapSegments = SNCDataBase::groupSegmentIntoProvince(segments);
    emit progressUpdated(2);
    QMap<QString ,int > map;

    for(int i=0;i<segments.size();i++)
    {
        QString  segment = segments.at(i);
        QString  city = mapSegments.value(segment);
        if(city.isEmpty())
            city = "未知地区";
        map.insert(city,map.value(city)+numbers.at(i).size());
        for(int j=0;j<numbers.at(i).size();j++)
        {
            mapCityPhone[city] += segment+numbers.at(i).at(j) + "\r\n";
        }
        emit progressUpdated(i+3);
    }

    foreach(QString city, mapCityPhone.keys())
    {
        QFile file(fileNamePrefix+"_"+city+"_"+QString::number(map.value(city))+".txt");
        file.open(QIODevice::WriteOnly);
        QTextStream out(&file);
        out<<mapCityPhone.value(city);
        file.close();
    }

    emit exportFinished();
}

/*
void NumberExport::exportByRegion()//export by city
{
    QStringList segments;
    QList<QStringList> numbers;
    segments = this->model->getSegments();
    numbers = this->model->getNumbers();
    emit progressMaxUpdated(segments.size()+2);
    emit progressUpdated(1);
    QMap<QString,QString> mapSegments ;
    if(this->isCity)
       mapSegments = SNCDataBase::groupSegmentIntoCity(segments);
    else
       mapSegments = SNCDataBase::groupSegmentIntoProvince(segments);
    emit progressUpdated(2);
    QMap<QString ,int > map;
    for(int i=0;i<segments.size();i++)
    {
        QString  segment = segments.at(i);
        QString  city = mapSegments.value(segment);
        if(city.isEmpty())
            city = "未知地区";
        map.insert(city,map.value(city)+numbers.at(i).size());
        QFile file(fileNamePrefix+"_"+city+".txt");
        file.open(QIODevice::Append);
        QTextStream out(&file);
        for(int j=0;j<numbers.at(i).size();j++)
        {
            if(i==segments.size()-1 && j==numbers.at(i).size()-1)
                out<<segment+numbers.at(i).at(j);
            else
                out<<segment+numbers.at(i).at(j)<<"\r\n";
        }
        file.close();
        emit progressUpdated(i+3);
    }
    foreach(QString city,map.keys())
    {
        QFile::rename(fileNamePrefix+"_"+city+".txt",fileNamePrefix+"_"+city+"_"+QString::number(map.value(city))+".txt");
    }
    emit exportFinished();
}*/