Posts

Scrapy Splash

In __init__ to run scraper from scrapy import cmdline cmdline.execute("scrapy crawl usps_service_spider".split()) Pipeline import pyodbc from scrapy.exceptions import NotConfigured class TrackingServicesPipeline(object): def __init__(self): pass def open_spider(self, spider): self.conn = pyodbc.connect("Driver={SQL Server Native Client 11.0};Server=192.168.168.40;Database=amz_reviews;UID=amz_reviews;PWD=amz@dev;port=1433;Integrated_Security=true;Trusted_Connection=no;") self.cursor = self.conn.cursor() def process_item(self, item, spider): # sql = "INSERT INTO table (field1, field2, field3) VALUES (%s, %s, %s)" # self.cursor.execute(sql, # ( # item.get("field1"), # item.get("field2"), # item.get("field3"), # ) # ...
Post a Comment
Read more

Stored procedure

AmazonListing USE [amz_reviews] GO /****** Object: StoredProcedure [dbo].[sp_AmazonListing] Script Date: 8/25/2020 12:01:52 PM ******/ SET ANSI_NULLS ON GO SET QUOTED_IDENTIFIER ON GO CREATE PROCEDURE [dbo].[sp_AmazonListing] @PageNumber int, @RowsPerPage int, @SortColumn NVARCHAR(30), @SortDirection NVARCHAR(10), @SearchWord NVARCHAR(150), @Channel VARCHAR(20), @Status int AS BEGIN select count(*) as Total from [amz_reviews].[dbo].[AmzListing] el where ( (el.Sku Like '%'+@SearchWord+'%' OR @SearchWord IS NULL Or @SearchWord = '') and (el.ChannelCode=@Channel OR @Channel IS NULL or @Channel='') and (el.Status=@Status OR @Status IS NULL or @Status=2) ) select el.Id,el.Title,el.Sku,el.Price,el.ChannelCode,el.Status,el.Url from AmzListing el where ( (el.Sku Like '%'+@SearchWord+'%' OR @SearchWord IS NULL Or @SearchWord = '') and (el.ChannelCode=@Channel OR @Channel IS NULL or ...
Post a Comment
Read more

MVC

Image
Create MVC project Web.config connectionStrings> add name="RDX_FBA_TRACKING" connectionString="Data Source=serverIP;Initial Catalog=dbName; user Id=USer; password=Pass; Integrated Security =False;" providerName="System.Data.SqlClient" /> /connectionStrings> AccountController using RDX.Domain.DTO; using RDX.Domain.Entities; using RDX.Infrastructure.Helper; using RDX.Infrastructure.Repository; using System; using System.Collections.Generic; using System.Linq; using System.Web; using System.Web.Mvc; namespace RDX.FBATracking.Controllers { public class AccountController : Controller { public ActionResult Login(string returnUrl) { if (returnUrl == null) returnUrl = "/"; ViewBag.ReturnUrl = returnUrl; return View(); } [HttpPost] public ActionResult Login(User model, string returnUrl) { try { ...
Post a Comment
Read more

Product Feed

ProductFeed using FashionExchange.Common.BLL; using FashionExchange.Common.DAL; using FashionExchange.Common.Enums; using FashionExchange.Common.Utils; using log4net; using System; using System.Collections.Generic; using System.Linq; using System.Net; using System.Text; namespace FashionExchange.Common.ProductFeeds { public abstract class ProductFeed { protected ILog log { get { return LogManager.GetLogger(this.GetType().Name); } } protected Store Store { get { return StoreManager.GetStoreByScraperClassName(this.GetType().Name); } } protected virtual bool ProductSizeOnSeparateRow { get { return false; } } private HashSet _existingProductNames = null; protected HashSet ExistingProductNames { get { if (_existingProductNames == null) {...
Post a Comment
Read more

Utility

StringUtil using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; namespace FashionExchange.Common.Utils { public class StringUtil { public static string SubstringFromStart(string value, string endStr, bool includeEndStr = false, bool ignoreCase = false) { int endIndex = ignoreCase ? value.IndexOf(endStr, StringComparison.CurrentCultureIgnoreCase) : value.IndexOf(endStr); if (endIndex == -1) return null; if (includeEndStr) endIndex += endStr.Length; return value.Substring(0, endIndex); } public static string SubstringToEnd(string value, string startStr, bool includeStartStr = false, bool ignoreCase = false) { int startIndex = ignoreCase ? value.IndexOf(startStr, StringComparison.InvariantCultureIgnoreCase) : value.IndexOf(startStr); if (startInde...
Post a Comment
Read more