DEDE 4.0 自動(dòng)采集源文件
放在網(wǎng)站跟目錄
WINDOWS系統(tǒng) 建立個(gè) ****.cmd 內(nèi)容
D:\php\php-cgi E:\wwwroot\**\cj.php
再做個(gè)計(jì)劃任務(wù)就可以了（D:\php\php-cgi E:\wwwroot\**\cj.php 知道是什么吧）
LINUX 系統(tǒng)定時(shí)間任務(wù)偶不會(huì) 一樣是 運(yùn)行 cj.php 會(huì)的可以回復(fù)下
cj.php 源文件
復(fù)制代碼 代碼如下:
<?php
//要放別的目錄請改路徑
require_once(dirname(__FILE__)."/include/config_base.php");
require_once(dirname(__FILE__)."/include/pub_collection.php");
require_once(dirname(__FILE__)."/include/pub_datalist.php");
require_once(dirname(__FILE__)."/include/inc_functions.php");
require_once(dirname(__FILE__)."/include/pub_dedetag.php");
require_once(dirname(__FILE__)."/include/inc_archives_view.php");
require_once(dirname(__FILE__)."/include/inc_arclist_view.php");
//添加數(shù)據(jù) 采集欄目名稱，采集欄目ID, 前臺(tái)欄目名稱,前臺(tái)欄目ID 有幾個(gè)就填寫幾行
$makearr = array();
array_push($makearr,array('采集一',1,'分類一',25));
array_push($makearr,array('采集二',2,'分類二',26));
//其它一些參數(shù)
$makehtml = 1 ;//生成文章
$threadnum = 5 ; //間隔時(shí)間
$pagesize = 5; //設(shè)置線程
$islisten = 0 ; //附加選項(xiàng) 不下載曾下載的網(wǎng)址-> 0 ; 僅下載未下載內(nèi)容->-1; 重新下載所有內(nèi)容->1
$sptime = 0 ; //防刷新的站點(diǎn)需設(shè)置 x秒
$channelid = 1 ;//導(dǎo)入的文章模型
$web = "www-data"; //web用戶
$webgroup = "www-data"; //web用戶組
$onlytitle = 1 ; //
while(sizeof($makearr)>0){
$nid = $makearr[0][1]; //采集欄目ID
$typeid = $makearr[0][3]; //前臺(tái)欄目ID
array_shift($makearr);
//采集
//多線程模式初次設(shè)置
$co = new DedeCollection();
$co->Init();
$co->LoadFromDB($nid);
$co->GetSourceUrl();
$co->dsql->SetQuery("Update dede_conote set lasttime='".mytime()."' where nid=$nid");
$co->dsql->ExecuteNoneQuery();
$co->dsql->SetQuery("Select aid,url From dede_courl where nid=$nid ");
$co->dsql->Execute(99);
while($row = $co->dsql->GetObject(99))
{
$lrow = $co->dsql->GetOne("Select * From dede_co_listenurl where url like
'".addslashes($row->url)."'");
if(is_array($lrow)){
if($islisten==0) continue;
}
else{
echo $inquery = "INSERT INTO dede_co_listenurl(nid,url) VALUES ('$nid',
'".addslashes($row->url)."');";
$co->dsql->ExecuteNoneQuery($inquery);
}
$co->DownUrl($row->aid,$row->url);
if($sptime>0) sleep($sptime);
}
$co->Close();
//導(dǎo)入到當(dāng)前的數(shù)據(jù)庫并生成html
$dsql = new DedeSql(false);
$maxidrow = $dsql->GetOne("Select max(ID) as maxid From dede_archives where typeid=$typeid
;");
$maxart_id = $maxidrow['maxid'];
$mrow = $dsql->GetOne("Select count(*) as dd From dede_courl where nid='$nid'");
$totalcc = $mrow['dd']; //
$rrow = $dsql->GetOne("Select typeid From dede_conote where nid='$nid'");
$ruleid = $rrow['typeid'];
$rrow = $dsql->GetOne("Select channelid From dede_co_exrule where aid='$ruleid'");
$channelid = $rrow['channelid'];
if(!isset($channelid)) $channelid = 0;
if(!isset($typeid)) $typeid = 0;
if(!isset($makehtml)) $makehtml = 0;
if(!isset($onlytitle)) $onlytitle = 1;
$dsql = new DedeSql(false);
$row = $dsql->GetOne("Select * From dede_co_exrule where aid='$ruleid'");
if(!is_array($row)){
echo "找不到導(dǎo)入規(guī)則，無法完成操作！";
$dsql->Close();
continue;
}
//分析規(guī)則，并生成臨時(shí)的SQL語句
$dtp = new DedeTagParse();
$dtp->LoadString($row['ruleset']);
$noteinfo = $dtp->GetTagByName('note');
$tablenames = explode(",",$noteinfo->GetAtt('tablename'));
$autofield = $noteinfo->GetAtt('autofield');
$synfield = $noteinfo->GetAtt('synfield');
$tablename1 = $tablenames[0];
$tb1SqlKey = "Insert Into $tablename1(";
$tb1SqlValue = " Values(";
if(count($tablenames)>=2){
$tablename2 = $tablenames[1];
$tb2SqlKey = "Insert Into $tablename2(";
$tb2SqlValue = " Values(";
if($synfield!=''){
$tb2SqlKey .= $synfield;
$tb2SqlValue .= "'@$synfield@'";
}
}
else{
$tablename2 = "";
$tb2SqlKey = "";
$tb2SqlValue = "";
}
$exKeys = Array();
foreach($dtp->CTags as $tagid => $ctag)
{
if($ctag->GetName()=='field')
{
$fieldname = $ctag->GetAtt('name');
$tbname = $ctag->GetAtt('intable');
if($tbname==$tablename1){
$tb1SqlKey .= ",$fieldname";
if($ctag->GetAtt('source')!='value'){
$tb1SqlValue .= ",'@#{$tbname}.{$fieldname}#@'";
}else{
$nvalue = str_replace('{tid}',$typeid,$ctag->GetInnerText
());
$nvalue = str_replace('{cid}',$channelid,$nvalue);
$nvalue = str_replace('{rank}',0,$nvalue);
$nvalue = str_replace('{admin}',"1",$nvalue);
$tb1SqlValue .= ",'$nvalue'";
}
}
else if($tbname==$tablename2){
$tb2SqlKey .= ",$fieldname";
if($ctag->GetAtt('source')!='value'){
$tb2SqlValue .= ",'@#{$tbname}.{$fieldname}#@'";
}else{
$nvalue = str_replace('{tid}',$typeid,$ctag->GetInnerText
());
$nvalue = str_replace('{cid}',$channelid,$nvalue);
$nvalue = str_replace('{rank}',0,$nvalue);
$tb2SqlValue .= ",'$nvalue'";
}
}
}
}
$tb1SqlKey = str_replace('(,','(',$tb1SqlKey).")";
$tb1SqlValue = str_replace('(,','(',$tb1SqlValue).");";
$tb1Sql = $tb1SqlKey.$tb1SqlValue;
if($tablename2!="")
{
$tb2SqlKey = str_replace("(,","(",$tb2SqlKey).")";
$tb2SqlValue = str_replace("(,","(",$tb2SqlValue).");";
$tb2Sql = $tb2SqlKey.$tb2SqlValue;
}
//導(dǎo)出數(shù)據(jù)的SQL操作
$dsql->SetQuery("Select * From dede_courl where nid='$nid' order by aid asc ");
$dsql->Execute();
while($row = $dsql->GetObject())
{
$tmpSql1 = $tb1Sql;
$tmpSql2 = $tb2Sql;
$dtp->LoadString($row->result);
$aid = $row->aid;
if(!is_array($dtp->CTags)){ continue; }
if($onlytitle){
$titletag = '';
foreach ($dtp->CTags as $ctag){
$tvalue = $ctag->GetAtt("name");
if($tvalue == 'dede_archives.title' || $tvalue ==
$cfg_dbprefix.'archives.title'){
$titletag = $ctag;
break;
}
}
if(is_object($titletag)){
$title = trim(addslashes($titletag->GetInnerText()));
$testrow = $dsql->GetOne("Select count(ID) as dd From dede_archives
where title like '%$title%'");
if($testrow['dd']>0){
echo "數(shù)據(jù)庫已存在標(biāo)題為: {$title} 的文檔，程序阻止了此內(nèi)
容導(dǎo)入<br/>";
continue;
}
}
}
foreach($dtp->CTags as $ctag)
{
if($ctag->GetName()!="field") continue;
$tvalue = $ctag->GetAtt("name");
$tmpSql1 = str_replace('@#'.$tvalue.'#@',addslashes($ctag->GetInnerText
()),$tmpSql1);
if($tablename2!=""){
$tmpSql2 = str_replace('@#'.$tvalue.'#@',addslashes($ctag-
>GetInnerText()),$tmpSql2);
}
}
$tmpSql1 = ereg_replace('@#(.*)#@','',$tmpSql1);
$rs = $dsql->ExecuteNoneQuery($tmpSql1);
if($rs && $tablename2!=""){
if($synfield!=""){
$lid = $dsql->GetLastID();
$tmpSql2 = str_replace("@$synfield@",$lid,$tmpSql2);
$rs = $dsql->ExecuteNoneQuery($tmpSql2);
if(!$rs) $dsql->ExecuteNoneQuery("Delete From $tablename1 where
$autofield='$lid'");
}
else $dsql->ExecuteNoneQuery($tmpSql2);
}
$dsql->ExecuteNoneQuery("update dede_courl set isex=1 where aid='$aid'");
}
$dsql->Close();
//生成html
if($channelid>0 && $makehtml==1){
if(!$maxart_id) $maxart_id =0;
$dsql = new DedeSql(false);
$dsql->SetQuery("Select ID From dede_archives where typeid=$typeid and ID
>$maxart_id ;");
$dsql->Execute();
while($row=$dsql->GetObject())
{
$ID = $row->ID;
$ac = new Archives($ID);
$rurl = $ac->MakeHtml();
$ac->Close();
}
$dsql->Close();
}
//生成首頁
$templet = "{style}/index.htm";
$templet = str_replace("{style}",$cfg_df_style,$templet);
$homeFile = dirname(__FILE__)."/index.html";
$homeFile = str_replace("\\","/",$homeFile);
$homeFile = str_replace("http://","/",$homeFile);
$fp = fopen($homeFile,"w") or die("你指定的文件名有問題，無法創(chuàng)建文件");
fclose($fp);
$pv = new PartView();
$pv->SetTemplet($cfg_basedir.$cfg_templets_dir."/".$templet);
$pv->SaveToHtml($homeFile);
$pv->Close();
}
$dsql = new DedeSql(false);
$dsql->SetQuery("Select ID,typedir From dede_arctype;");
$dsql->Execute();
while($row=$dsql->GetObject())
{
//生成列表html
$lv = new ListView($row->ID);
$lv->MakeHtml();
$lv->Close();
}
$dsql->Close();
?>
