Welcome

This blog is only for me!

Comments

Post a Comment

Popular posts from this blog

Web Scraping material

Scraper using System; using System.Collections.Generic; using System.Collections.Concurrent; using System.Linq; using System.Text; using log4net; using FashionExchange.Common.DAL; using FashionExchange.Common.BLL; using HtmlAgilityPack; using System.Net; using System.Threading; using FashionExchange.Common.Utils; using System.IO; using System.Web.Hosting; using FashionExchange.Common.Enums; using OpenQA.Selenium.Remote; namespace FashionExchange.Common.Scrapers { public abstract class Scraper { protected ILog log { get { return LogManager.GetLogger(this.GetType().Name); } } public Store Store { get { return StoreManager.GetStoreByScraperClassName(this.GetType().Name); } } protected SiteMap _siteMap = null; protected virtual SiteMap SiteMap { get { if (_siteMap == null) _siteMap = new SiteMap(Store.Url);...
Read more

Scrapy Splash

In __init__ to run scraper from scrapy import cmdline cmdline.execute("scrapy crawl usps_service_spider".split()) Pipeline import pyodbc from scrapy.exceptions import NotConfigured class TrackingServicesPipeline(object): def __init__(self): pass def open_spider(self, spider): self.conn = pyodbc.connect("Driver={SQL Server Native Client 11.0};Server=192.168.168.40;Database=amz_reviews;UID=amz_reviews;PWD=amz@dev;port=1433;Integrated_Security=true;Trusted_Connection=no;") self.cursor = self.conn.cursor() def process_item(self, item, spider): # sql = "INSERT INTO table (field1, field2, field3) VALUES (%s, %s, %s)" # self.cursor.execute(sql, # ( # item.get("field1"), # item.get("field2"), # item.get("field3"), # ) # ...
Read more

Utility

StringUtil using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; namespace FashionExchange.Common.Utils { public class StringUtil { public static string SubstringFromStart(string value, string endStr, bool includeEndStr = false, bool ignoreCase = false) { int endIndex = ignoreCase ? value.IndexOf(endStr, StringComparison.CurrentCultureIgnoreCase) : value.IndexOf(endStr); if (endIndex == -1) return null; if (includeEndStr) endIndex += endStr.Length; return value.Substring(0, endIndex); } public static string SubstringToEnd(string value, string startStr, bool includeStartStr = false, bool ignoreCase = false) { int startIndex = ignoreCase ? value.IndexOf(startStr, StringComparison.InvariantCultureIgnoreCase) : value.IndexOf(startStr); if (startInde...
Read more