• 设为首页
  • 点击收藏
  • 手机版
    手机扫一扫访问
    迪恩网络手机版
  • 关注官方公众号
    微信扫一扫关注
    公众号
登陆 注册

C#获取网页信息并存入数据库

原作者: [db:作者] 来自: [db:来源] 收藏 邀请

1，获取以及商品分类信息

给一网页获取网页上商品信息的分类

using Skay.WebBot;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading;
using System.Windows.Forms;
using Ivony.Html;
using Ivony.Html.Parser;
using System.Data.SqlClient;

namespace catchGoods
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }
        public static Thread th;
        private void button1_Click(object sender, EventArgs e)
        {
            th = new Thread(GetJDData);
            th.Start();
         }
        public void GetJDData()
        {
            SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");
            conn.Open();

            string str = "http://www.htluxe.com";
            HttpUtility http = new HttpUtility();
            string html = http.GetHtmlText(str);
            var documenthtml = new JumonyParser().Parse(html);
            var items = documenthtml.Find(".categroup dl");
            foreach(var item in items)
            {
                string name = item.FindFirst("h4 a").InnerText();
                string remarkOdd = item.FindFirst("h4 a").Attribute("href").Value();
                string remark = remarkOdd.Split('=')[1];
                this.Invoke((EventHandler)(delegate
                {
                    listBox1.Items.Add(name);

                }));//有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼（委托？
                string into = string.Format("insert into exerciseOneSort (className, remark) values ('" + name + "', '" + remark + "')");
                SqlCommand com = new SqlCommand(into, conn);
                int i = com.ExecuteNonQuery();

                var elements = item.Find("dt p a");
                foreach(var element in elements)
                {
                    string nameTwo = element.InnerText();
                    string url = "http://www.htluxe.com/" + element.Attribute("href").Value();
                    string intoTwo = string.Format("insert into exerciseTwoSort (className, url, idplus) values ('" + nameTwo + "', '" + url + "', '" + remark + "')");
                    SqlCommand comTwo = new SqlCommand(intoTwo, conn);
                    int j = comTwo.ExecuteNonQuery();
                }
            }
        }
    }
}

 完整版

using Skay.WebBot;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading;
using System.Windows.Forms;
using Ivony.Html;
using Ivony.Html.Parser;
using System.Data.SqlClient;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json;

namespace catchGoods
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }
        public static Thread th;
        private void button1_Click(object sender, EventArgs e)
        {
            th = new Thread(GetJDDataOne);
            th.Start();
            //SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");
            //conn.Open();
            //string str = string.Format("delete from exerciseTwoSort");
            //SqlCommand com = new SqlCommand(str, conn);
            //int w = com.ExecuteNonQuery();
         }
        public void GetJDDataOne()
        {
            SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");
            conn.Open();

            string str = "http://www.htluxe.com";
            HttpUtility http = new HttpUtility();
            string html = http.GetHtmlText(str);
            var documenthtml = new JumonyParser().Parse(html);
            var items = documenthtml.Find(".categroup dl");
            foreach(var item in items)
            {
                string name = item.FindFirst("h4 a").InnerText();
                string remarkOdd = item.FindFirst("h4 a").Attribute("href").Value();
                string remark = remarkOdd.Split('=')[1];
                this.Invoke((EventHandler)(delegate
                {
                    listBox1.Items.Add(name+"  "+remark);

                }));//有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼
                string into = string.Format("insert into exerciseOneSort (className, remark) values ('" + name + "', '" + remark + "')");
                SqlCommand com = new SqlCommand(into, conn);
                int i = com.ExecuteNonQuery();

                var elements = item.Find("dt p a");
                foreach(var element in elements)
                {
                    string nameTwo = element.InnerText();
                    string url = "http://www.htluxe.com/" + element.Attribute("href").Value();
                    this.Invoke((EventHandler)(delegate
                    {
                        listBox1.Items.Add(nameTwo + "  " +url + "  " + remark);

                    }));//有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼
                    string intoTwo = string.Format("insert into exerciseTwoSort (className, url, idplus) values ('" + nameTwo + "', '" + url + "', '" + remark + "')");
                    SqlCommand comTwo = new SqlCommand(intoTwo, conn);
                    int j = comTwo.ExecuteNonQuery();
                }
            }
        }
        int page = 0;
        string surl;
        public static Thread th2;
        private void button2_Click(object sender, EventArgs e)
        {
            listBox1.Items.Clear();
            th2 = new Thread(threadTwo);
            th2.Start();
            //SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");
            //conn.Open();
            //string str = string.Format("delete from GoodsList");
            //SqlCommand com = new SqlCommand(str, conn);
            //int d = com.ExecuteNonQuery();
            //MessageBox.Show(Convert.ToString(d));
        }
        public void threadTwo()
        {
            SqlConnection conn = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");
            conn.Open();
            //如果字符串中含有单引号，解决方法1----------------------------------
            //string titlestr = "念佛'夜晚访'问欧诺'法";
            //string pricestr = "99.00";
            //string sqlstr = string.Format("insert into goods (name,price) values (@name,'" + pricestr + "')");
            //SqlCommand sqlcom = new SqlCommand(sqlstr, conn);
            //sqlcom.Parameters.Add("@name", titlestr);
            //sqlcom.ExecuteNonQuery();
            //解决方法2-----------------------------------------------------------------------
            //string bufffuck = "fdgjjf'fgfgf";
            //bufffuck = bufffuck.Replace("'", "''");
            //string sqlstr = string.Format("insert into goods (name) values ('"+bufffuck+"')");
            //SqlCommand sqlcom = new SqlCommand(sqlstr, conn);
            //int y = sqlcom.ExecuteNonQuery();


            string sel = string.Format("select url from exerciseTwoSort");
            DataTable dt = new DataTable();
            SqlDataAdapter dapt = new SqlDataAdapter(sel, conn);
            dapt.Fill(dt);

            for (int i = 0; i < dt.Rows.Count; i++)
            {
                surl = dt.Rows[i][0].ToString();
                HttpUtility httpTwo = new HttpUtility();
                string htmlTwo = httpTwo.GetHtmlText(surl);
                var documenthtml = new JumonyParser().Parse(htmlTwo);
                var pageto = Convert.ToString(documenthtml.FindFirst(".goods-page-min label").InnerText());
                page = Convert.ToInt32(pageto.Split('/')[1]);
                GetJDData();
            }
        }
         void GetJDData()
         {
              for (int j = 1; j <= page; j++)
              {
                  string htmlTwo = surl + "&price_min=0&price_max=0&page=" + j + "&sort=sort_order%20asc,last_update&order=DESC";
                  HttpUtility httpMid = new HttpUtility();
            string htmlMid = httpMid.GetHtmlText(htmlTwo);
            var documenthtmlMid = new JumonyParser().Parse(htmlMid);
            var items = documenthtmlMid.Find(".piclist li");
            foreach(var item in items)
            {
                string title = item.FindFirst(".base a").InnerText();
                title = title.Replace("'", "''");
                //string goodsurl = "http://www.htluxe.com/"+item.FindFirst(".base a").Attribute("href").Value();
                //string subhtml = http.GetHtmlText(goodsurl, "utf-8", "text/html; charset=utf-8");
                //string Area_Html = http.GetHtmlText(goodsurl.Split('?')[0] + "?act=price&" + goodsurl.Split('?')[1], "utf-8", "text/html;charset=utf-8", "");
                try
                {                     
                    string nowPrice = item.FindFirst(".minprice").InnerText();
                    string oldPrice = item.FindFirst(".maxprice").InnerText();
                    string popular = item.FindFirst(".ratecount strong").InnerText();
                    string sales = item.FindFirst(".soldnum strong").InnerText();
                    string contents = item.FindFirst(".commentcount strong").InnerText().ToString();
                    string htmlThree = "http://www.htluxe.com/" + item.FindFirst("dt a").Attribute("href").Value().ToString();
                    HttpUtility httpThree = new HttpUtility();
                    string htmlBuff = httpThree.GetHtmlText(htmlThree);
                    var documenthtmlThree = new JumonyParser().Parse(htmlBuff);
                    string sben = documenthtmlThree.FindFirst(".promotionMiddleTop p").InnerText().ToString();
                    string num = sben.Split('：')[1];
                   
                       string starLevel = documenthtmlThree.FindFirst(".m-ratescore i").InnerText().ToString();
                       bufff(title, nowPrice, oldPrice, popular, sales, num, contents, starLevel);
                       this.Invoke((EventHandler)(delegate
                       {
                           listBox1.Items.Add(title + "  " + nowPrice + "  " + num + "  " + oldPrice + "  " + sales + "  " + popular + " " + contents + " " + starLevel);

                       }));
                        //有线程时listbox添加东西的时候要这么写不然报错谁知道什么鬼
                    //this.listBox1.Items.Add("");
                    //listBox1.Items.Add(title + "  " + nowPrice + "  " + num + "  " + oldPrice + "  " + sales + "  " + popular);                   
                       
                }
                catch
                {
                    MessageBox.Show("异常");
                }
                
            }

              }     
        }
        private static void bufff(string title, string nowPrice, string oldPrice,
            string popular, string sales, string num, string contents, string starLevel)
        {
            SqlConnection conn2 = new SqlConnection("Data Source=.;Initial Catalog=StuTinafirst;User ID=sa;Password=123456");
            conn2.Open();

            string strstr = string.Format("insert into GoodsList (name, num, sales, popular, starLevel, contents, price, oldPrice) values ('" + title + "', '" +num + "',  '" + sales + "', '" + popular + "', '"+starLevel+"', '"+contents+"', '" + nowPrice + "', '" + oldPrice + "')");
            SqlCommand com2 = new SqlCommand(strstr, conn2);
            int g = com2.ExecuteNonQuery();
        }
    }
}

 

鲜花
握手
雷人
路过
鸡蛋
该文章已有0人参与评论

请发表评论

全部评论

专题导读
More+
上一篇：
VSTO：使用C#开发Excel、Word【3】发布时间：2022-07-13
下一篇：
opencv直线检测在c#、Android和ios下的实现方法发布时间：2022-07-13
热门推荐
More+
热门话题
More+
阅读排行榜

扫描微信二维码

查看手机版网站

随时了解更新最新资讯

139-2527-9053

在线客服（服务时间 9:00～18:00）

在线QQ客服
地址：深圳市南山区西丽大学城创智工业园
电邮：jeky_zhao#qq.com
移动电话：139-2527-9053

Powered by 互联科技 X3.4© 2001-2213 极客世界.|Sitemap