联调后台

app/controller/home.js

@@ -179,7 +179,9 @@ class HomeController extends Controller {
         .then(res => {
           // saveFile(`goods.json`, JSON.stringify(res, null, 4), './output/goods/');
           console.log('获取多个商品详情成功');
-        });
+        }).catch(err => {
+          console.log('获取多个商品详情失败');
+        })
       data.success = true;
       data.message = '等待回调';
     } else {

app/service/get_goods.js

@@ -6,6 +6,7 @@ const { baseData } = require('../../tools/base-data');
 const { skuData } = require('../../tools/sku-data');
 const login = require('../../spider/login');
 const { saveFile } = require('../../tools/save-file');
+const { httpPost } = require('../../tools/http-post');
 
 class getGoodDetailService extends Service {
 
@@ -25,12 +26,15 @@ class getGoodDetailService extends Service {
 
             async function eachUrl() {
 
-                const url = self.urls[self.index];
+                let url = self.urls[self.index];
+                if(url.indexOf('//') === 0){
+                    url = 'https:'+url;
+                }
             
                 goodDetailLogger.info(`开始获取详情：${url}`);
 
                 await driver.switchTo().newWindow('tab');
-                await driver.get('https://'+url);
+                await driver.get(url);
                 await driver.sleep(3000);
 
                 // 自动滚动 到 y=20000
@@ -97,7 +101,8 @@ class getGoodDetailService extends Service {
     }
 
     async callback(data) {
-        console.log(`完成:${data.title}`);
+        console.log(`发送后台完成:  ${data.title}`);
+        httpPost('/Migration/GetGoodDetailCallBack',data);
         saveFile(`${data.title}.json`, JSON.stringify(data, null, 4), './output/goods/');
     }
 

cookie/cookie.json

@@ -1,38 +1,48 @@
 [
     {
         "domain": ".tmall.com",
-        "expiry": 1614675134,
+        "expiry": 1614680704,
         "httpOnly": false,
-        "name": "tfstk",
+        "name": "l",
         "path": "/",
         "secure": false,
-        "value": "c4BNB7tX7ReZoxpbwd9qhACmTCsOwtT2yv-yssY9e1WOnhfDbqK3AqKe0UeHs"
+        "value": "eBgy8EDqOsL7HdgDBO5ZPurza77tiIRb4sPzaNbMiInca6CFNFw-RNQ4HTh9ldtjgtCX7etr5xfW3RLHR3VcCU0ifdF-nHrE3xvO."
     },
     {
-        "domain": "list.tmall.com",
+        "domain": ".tmall.com",
+        "expiry": 1614680704,
         "httpOnly": false,
-        "name": "res",
+        "name": "tfstk",
         "path": "/",
         "secure": false,
-        "value": "scroll%3A1519*5419-client%3A1519*711-offset%3A1519*5419-screen%3A1536*865"
+        "value": "cRAcBNfrZKWbUwWkdj1XRdiYtxAGZFeVvCJeaQAvhu5ymCOPiNezLxaxrNKD6R1.."
     },
     {
-        "domain": ".tmall.com",
-        "expiry": 1614675134,
+        "domain": "lumn.tmall.com",
+        "expiry": 1601720703,
         "httpOnly": false,
-        "name": "l",
+        "name": "pnm_cku822",
         "path": "/",
         "secure": false,
-        "value": "eB_0f0ZuOsq93P_bBOfanurza77OSIRYYuPzaNbMiOCP90CB5J4CWZyv8qY6C3GVh6kvRuusn_92BeYBqQAonxvttBALurkmn"
+        "value": ""
+    },
+    {
+        "domain": ".tmall.com",
+        "httpOnly": true,
+        "name": "uc4",
+        "path": "/",
+        "sameSite": "None",
+        "secure": true,
+        "value": "nk4=0%40AgbxYJ9INhjQhPNK54DKNy4%3D&id4=0%40U2grGNp7s350SOTPb%2BaHY7EXvJ239nnL"
     },
     {
-        "domain": "list.tmall.com",
-        "expiry": 1601715134,
+        "domain": ".tmall.com",
         "httpOnly": false,
-        "name": "pnm_cku822",
+        "name": "_tb_token_",
         "path": "/",
-        "secure": false,
-        "value": "098%23E1hvRpvUvbpvUvCkvvvvvjiPnLchAj3EPsFpsjljPmP91jtURL5WsjDRRFMOzjrERphvChCvvvvCvpvVphhvvvvvKphv8hCvCEZvvhe0phvOvvvvpIxvpC9vvvC2UyCvVvvvvhnTphvOmvvvpzkEvpvVpyUUCCA4uphvmhCvCnViHawAmphvLhHVsvmFeeQ4b64B9CkaU6UsxI2h5C6AxsBCAfyp%2B3%2BFjomHff8rwkGaW4c6%2BulgEcYR1nvOe5inwy29%2B2Kz8Zl9ZRAn%2BbyDCcEmTWyPvpvhvv2MMTwCvvBvpvpZ"
+        "sameSite": "None",
+        "secure": true,
+        "value": "5fe0065b3fee7"
     },
     {
         "domain": ".tmall.com",
@@ -59,7 +69,7 @@
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "1a90fcb04988b900c09059a31730aca6"
+        "value": "1c42177844636cc3248159da13468845"
     },
     {
         "domain": ".tmall.com",
@@ -68,7 +78,7 @@
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "nk2=AGioONbo&lg2=W5iHLLyFOGW7aA%3D%3D&vt3=F8dCufXH8BiMNVfzsDc%3D&id2=UUphw2VSPKDoJSjKzw%3D%3D"
+        "value": "vt3=F8dCufXH8xuFNtxfnbo%3D&id2=UUphw2VSPKDoJSjKzw%3D%3D&lg2=URm48syIIVrSKA%3D%3D&nk2=AGioONbo"
     },
     {
         "domain": ".tmall.com",
@@ -77,63 +87,43 @@
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "existShop=false&cookie16=UtASsssmPlP%2Ff1IHDsDaPRu%2BPw%3D%3D&cookie21=Vq8l%2BKCLjA%2Bl&pas=0&cookie14=UoTV5Y00TGloQA%3D%3D&cookie15=Vq8l%2BKCLz3%2F65A%3D%3D"
-    },
-    {
-        "domain": ".list.tmall.com",
-        "httpOnly": false,
-        "name": "res",
-        "path": "/",
-        "sameSite": "Lax",
-        "secure": true,
-        "value": "scroll%3A1019*6478-client%3A1019*675-offset%3A1019*6478-screen%3A1536*864"
-    },
-    {
-        "domain": ".list.tmall.com",
-        "expiry": 1914482989,
-        "httpOnly": false,
-        "name": "_med",
-        "path": "/",
-        "sameSite": "Lax",
-        "secure": true,
-        "value": "dw:1228.8&dh:692&pw:1536&ph:865&ist:0"
+        "value": "pas=0&cookie16=V32FPkk%2FxXMk5UvIbNtImtMfJQ%3D%3D&cookie15=V32FPkk%2Fw0dUvg%3D%3D&cookie21=U%2BGCWk%2F7pY%2FF&existShop=false&cookie14=UoTV5Y00R2Yyew%3D%3D"
     },
     {
         "domain": ".tmall.com",
-        "httpOnly": true,
-        "name": "sgcookie",
+        "httpOnly": false,
+        "name": "dnk",
         "path": "/",
-        "sameSite": "Lax",
+        "sameSite": "None",
         "secure": true,
-        "value": "EVUqFItUf0bj6SKWo6mZf"
+        "value": "cr_150"
     },
     {
         "domain": ".tmall.com",
-        "expiry": 2229843134,
         "httpOnly": false,
-        "name": "cna",
+        "name": "t",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "uZzXFx/7rVgCATz/5/mSn7qI"
+        "value": "ca89363f3b36283deee03d4b1c73011e"
     },
     {
         "domain": ".tmall.com",
         "httpOnly": false,
-        "name": "login",
+        "name": "_nk_",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "true"
+        "value": "cr_150"
     },
     {
         "domain": ".tmall.com",
-        "expiry": 1614675134,
+        "expiry": 1614680703,
         "httpOnly": false,
         "name": "isg",
         "path": "/",
         "secure": false,
-        "value": "BJWVwBRNZoQtSEJ4KVOPXcB6pJFPkkmkV2xfORc6UYxbbrVg3-JZdKMsPHJY9WFc"
+        "value": "BDw8QNl0T5_WTHvDAtREjenNDdruNeBfJvOG4ha9zScK4dxrPkWz7kzTxQmZqRi3"
     },
     {
         "domain": ".tmall.com",
@@ -146,141 +136,115 @@
     },
     {
         "domain": ".tmall.com",
-        "httpOnly": false,
-        "name": "dnk",
+        "httpOnly": true,
+        "name": "cookie17",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "cr_150"
+        "value": "UUphw2VSPKDoJSjKzw%3D%3D"
     },
     {
         "domain": ".tmall.com",
-        "httpOnly": false,
-        "name": "t",
+        "httpOnly": true,
+        "name": "cookie1",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "e0b1ea2c43dc88bbee67f47020f41362"
+        "value": "UoYY5pPX%2F4qYXmjLO4nDTKvKkulnxBNok4hiG9327QA%3D"
     },
     {
         "domain": ".tmall.com",
-        "httpOnly": false,
-        "name": "_nk_",
+        "expiry": 1914241563,
+        "httpOnly": true,
+        "name": "enc",
         "path": "/",
-        "sameSite": "None",
+        "sameSite": "Lax",
         "secure": true,
-        "value": "cr_150"
+        "value": "MaT95uFYDGynG8cKXSw0DhPeUMNmbBVzDipL%2BsJ2d6OJdKtPAq76bQ3M3qnP0mrcY3kAYns9SfO5dl7OBx4G10EDU9wbzUPYWltFX6V4irM%3D"
     },
     {
         "domain": ".tmall.com",
-        "expiry": 1630687933,
+        "expiry": 1599209128,
         "httpOnly": false,
-        "name": "lid",
+        "name": "xlly_s",
         "path": "/",
-        "sameSite": "None",
+        "sameSite": "Lax",
         "secure": true,
-        "value": "cr_150"
+        "value": "1"
     },
     {
         "domain": ".tmall.com",
-        "httpOnly": true,
-        "name": "cookie17",
+        "httpOnly": false,
+        "name": "csg",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "UUphw2VSPKDoJSjKzw%3D%3D"
+        "value": "c347b6b9"
     },
     {
         "domain": ".tmall.com",
         "httpOnly": false,
-        "name": "_tb_token_",
+        "name": "tracknick",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "7e7986d35e68a"
+        "value": "cr_150"
     },
     {
         "domain": ".tmall.com",
-        "httpOnly": true,
-        "name": "uc4",
+        "httpOnly": false,
+        "name": "login",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "nk4=0%40AgbxYJ9INhjQhPNK54tsnbI%3D&id4=0%40U2grGNp7s350SOTPb%2BaHY7EXvJ28jK%2Fk"
+        "value": "true"
     },
     {
         "domain": ".tmall.com",
-        "expiry": 1914241563,
-        "httpOnly": true,
-        "name": "enc",
-        "path": "/",
-        "sameSite": "Lax",
-        "secure": true,
-        "value": "MaT95uFYDGynG8cKXSw0DhPeUMNmbBVzDipL%2BsJ2d6OJdKtPAq76bQ3M3qnP0mrcY3kAYns9SfO5dl7OBx4G10EDU9wbzUPYWltFX6V4irM%3D"
-    },
-    {
-        "domain": "list.tmall.com",
-        "expiry": 1914483134,
+        "expiry": 2229848698,
         "httpOnly": false,
-        "name": "_med",
-        "path": "/",
-        "secure": false,
-        "value": "dw:1536&dh:865&pw:1920&ph:1081.25&ist:0"
-    },
-    {
-        "domain": ".tmall.com",
-        "httpOnly": true,
-        "name": "cookie1",
+        "name": "cna",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "UoYY5pPX%2F4qYXmjLO4nDTKvKkulnxBNok4hiG9327QA%3D"
+        "value": "ebLXF7351nwCATz/5/maVKrZ"
     },
     {
         "domain": ".tmall.com",
-        "expiry": 1599209128,
-        "httpOnly": false,
-        "name": "xlly_s",
+        "httpOnly": true,
+        "name": "sgcookie",
         "path": "/",
         "sameSite": "Lax",
         "secure": true,
-        "value": "1"
+        "value": "EjcgFvxRgaK7bqlyMzg6M"
     },
     {
         "domain": ".tmall.com",
         "httpOnly": false,
-        "name": "csg",
+        "name": "_l_g_",
         "path": "/",
         "sameSite": "None",
         "secure": true,
-        "value": "2a0f3720"
+        "value": "Ug%3D%3D"
     },
     {
         "domain": ".tmall.com",
+        "expiry": 1630693503,
         "httpOnly": false,
-        "name": "tracknick",
+        "name": "lid",
         "path": "/",
         "sameSite": "None",
         "secure": true,
         "value": "cr_150"
     },
     {
-        "domain": ".tmall.com",
-        "httpOnly": false,
-        "name": "_l_g_",
-        "path": "/",
-        "sameSite": "None",
-        "secure": true,
-        "value": "Ug%3D%3D"
-    },
-    {
-        "domain": ".list.tmall.com",
-        "expiry": 1601515957,
+        "domain": ".lumn.tmall.com",
+        "expiry": 1601717040,
         "httpOnly": false,
         "name": "pnm_cku822",
         "path": "/",
         "sameSite": "Lax",
         "secure": true,
-        "value": "098%23E1hvZpvUvbpvUpCkvvvvvjiPnLLyAj3CRLS9AjrCPmPWsj1hRsFpsj1ER25WsjnbR9GCvvLMMQvvvphvC9mvphvvvbyCvmFMM6CYMMMYp9vvBhgvvvHBvvC8o9vv9Dvvvhi8vvmmPvvvBHgvvUVouphvmvvvpLZabLRJkphvC9hvpyP9lvyCvhQmQNj1jaoK5F%2FAVAdWaB4AdX3sbSoxdX9fdeO2fwo4d34AVAdZaXgM%2Bul1p57Q%2Bu0O5C69D70Oeug7Ecqhs8TJVXilZtxreEuKHkx%2FRphvCvvvphm5vpvhvvCCBv%3D%3D"
+        "value": ""
     }
 ]

spider/login.js

@@ -83,7 +83,7 @@ async function login(opts = {}) {
 // 自动登录
 async function autoLogin(driver) {
 
-    await driver.get('https://login.taobao.com/member/login.jhtml?tpl_redirect_url=https%3A%2F%2Flist.tmall.com%2Fsearch_product.htm%3Fq%3D41%26imgfile%3D%26commend%3Dall%26ssid%3Ds5-e%26search_type%3Dtmall%26sourceId%3Dtb.index%26spm%3Da21bo.2017.201856-taobao-item.1%26ie%3Dutf8%26initiative_id%3Dtbindexz_20170306&style=miniall&enup=true&newMini2=true&full_redirect=true&sub=true&from=tmall&allp=assets_css%3D3.0.10/login_pc.css&pms=1598880835545');
+    await driver.get('https://login.taobao.com/member/login.jhtml?tpl_redirect_url=https%3A%2F%2Flumn.tmall.com%2Fsearch.htm%3Fspm%3Da1z10.3-b-s.w5002-15438998724.1.48a63b51L03f6W%26search%3Dy&style=miniall&enup=true&newMini2=true&full_redirect=true&sub=true&from=tmall&allp=assets_css%3D3.0.10/login_pc.css&pms=1599124573437');
 
     // 检测登录框
     await driver.wait(webdriver.until.elementLocated(webdriver.By.xpath('//*[@id="fm-login-id"]')), 60000, '未检测到登录框');

spider/tmall_get_list_data.js

@@ -6,8 +6,6 @@ function tmallGetListData(htmlStr) {
 
     let $ = cheerio.load(htmlStr);
     let list = $('#J_ShopSearchResult .J_TItems>div');
-    console.log(`开始获取列表: 共${list.length}个数据`)
-
     for (var i = 0; i < list.length; i++) {
         const item = list[i];
         // 遇到分页直接停止，分页后面是推荐的
@@ -26,17 +24,7 @@ function tmallGetListData(htmlStr) {
             });
         }
     }
-    // list.splice(list.length - 8);
-    // list.each((i, e) => {
-    //     const title = $(e).find('.item-name').text();
-    //     const img = $(e).find('.photo img').attr('src');
-    //     const url = $(e).find('.item-name').attr('href');
-    //     const id = url.replace(/(.+id=)(\d+)(&.+)/igm, '$2');
-    //     const obj = {
-    //         title, img, url, id
-    //     }
-    //     goodsArr.push(obj)
-    // });
+    console.log(`列表获取完成: 共${goodsArr.length}个数据`)
     return goodsArr;
 }
 

test.js

@@ -1,43 +1,19 @@
-var http = require('http');
-var querystring = require('querystring');
-var data = [
-    {
-        "title": "施耐德电气墙壁电源开关插座附件防水面板盖防水盒 防溅盒E223DV ",
-        "img": "//img.alicdn.com/bao/uploaded/i1/1132886103/TB1jabhklcHL1JjSZFBXXaiGXXa_!!0-item_pic.jpg_180x180.jpg",
-        "url": "//detail.tmall.com/item.htm?id=17369050091&rn=ce41efca7f7d5fcd10332095b5dba323&abbucket=8",
-        "id": "17369050091"
-    },
-    {
-        "title": "施耐德电气开关插座面板一开二三插四极五孔插座单联双控 如意白 ",
-        "img": "//img.alicdn.com/bao/uploaded/i2/1132886103/O1CN01JCELj31uxDgCq5xQb_!!0-item_pic.jpg_180x180.jpg",
-        "url": "//detail.tmall.com/item.htm?id=17784266184&rn=ce41efca7f7d5fcd10332095b5dba323&abbucket=8",
-        "id": "17784266184"
-    }
-];
 
-var post_data = JSON.stringify(data);
-console.log(post_data,'111');
+// 后台列表保存测试
 
-var options = {
-    host: 'xcx.chuangzhikeji.com',
-    port: 80,
-    path: '/Product/GetGoodListCallBack',
-    method: 'POST',
-    headers:{
-        'Content-Type':'application/json'
-    }
-};
+// const fs = require('fs');
+// const { httpPost } = require('./tools/http-post');
+// // 判断是否有cookie
+// const arr = fs.readFileSync('./output/goodList/goodList.json', 'utf8');
+// let arr2 = JSON.parse(arr)
+// console.log(arr2.length);
+// httpPost('/Migration/GetGoodListCallBack', arr2)
 
 
-var req = http.request(options, function (res) {
-    console.log('STATUS: ' + res.statusCode);
-    console.log('HEADERS: ' + JSON.stringify(res.headers));
-    res.setEncoding('utf8');
-    res.on('data', function (chunk) {
-        console.log('BODY: ' + chunk);
-    });
-});
-
-// write data to request body
-req.write(post_data);
-req.end();
+const fs = require('fs');
+const { httpPost } = require('./tools/http-post');
+// 判断是否有cookie
+const obj = fs.readFileSync('./output/goods/a.json', 'utf8');
+let obj2 = JSON.parse(obj)
+console.log(obj2.length);
+httpPost('/Migration/GetGoodDetailCallBack', obj2)

test.txt

@@ -23,5 +23,10 @@ http://127.0.0.1:7001/getGoodDetail?url=https%3A%2F%2Fdetail.tmall.com%2Fitem.ht
 http://127.0.0.1:7001/getGoodList?url=https%3A%2F%2Fdetail.tmall.com%2Fitem.htm%3Fspm%3Da220m.1000858.1000725.36.4d3457ddDpsZdy%26id%3D611982420654%26skuId%3D4312674497656%26user_id%3D468175467%26cat_id%3D2%26is_b%3D1%26rn%3Db64961bddcea2b495376d2428795e257
 
 
+// 请求多商品详情
+http://127.0.0.1:7001/getGoods
+[
+	"https://detail.tmall.com/item.htm?spm=a1z10.3-b-s.w4011-15438998729.282.b3c13b519I24yH&id=594298803350&rn=4d154b9e4f2063b310447791cb1a430b&abbucket=19"
+]
 
 /getGoods     post请求 

tools/http-post.js

@@ -20,12 +20,16 @@ function httpPost(path,data){
     
     
     var req = http.request(options, function (res) {
-        console.log('STATUS: ' + res.statusCode);
-        console.log('HEADERS: ' + JSON.stringify(res.headers));
-        res.setEncoding('utf8');
-        res.on('data', function (chunk) {
-            console.log('BODY: ' + chunk);
-        });
+        if(res.statusCode === 200){
+            console.log('发送后台数据成功')
+        }else{
+            console.log('STATUS: ' + res.statusCode);
+            console.log('HEADERS: ' + JSON.stringify(res.headers));
+            res.setEncoding('utf8');
+            res.on('data', function (chunk) {
+                console.log('BODY: ' + chunk);
+            });
+        }
     });
     
     // write data to request body

tools/sku-data.js

@@ -10,7 +10,8 @@ const { saveFile } = require('./save-file');
  */
 function skuData(driver) {
 
-    let sku = [];
+    let sku = [];   // 输出用
+    let codeSku = [];   // 代码用
 
     return new Promise(async (resolve, reject) => {
 
@@ -23,7 +24,13 @@ function skuData(driver) {
             const list = await item.findElements(webdriver.By.tagName("li:not(.tb-out-of-stock)"));
             let obj = {
                 title,
-                list: []
+                list: [],
+            }
+            let objCode = {
+                title,
+                list: [],
+                index:0,
+                num:list.length
             }
             for (let item2 of list) {
                 const nameEle = await item2.findElement(webdriver.By.tagName('span'))
@@ -33,13 +40,21 @@ function skuData(driver) {
                 img = img ? img.replace(/(.+)\(([^\>)]+)\)(.+)/igm, '$2') : '';
                 const itemObj = {
                     title: name,
-                    img
+                    img,
+                }
+                const itemObjCode = {
+                    title: name,
+                    img,
+                    ele:item
                 }
                 obj.list.push(itemObj)
+                objCode.list.push(itemObjCode)
 
             }
             sku.push(obj);
+            codeSku.push(objCode);
         }
+        
 
         resolve(sku);