using System; using System.Collections.Generic; using System.Drawing; using System.Linq; using System.Net; using System.Threading.Tasks; using HtmlAgilityPack; using RyzStudio.Net; namespace BookmarkManager.Services { public class WebProvider { private readonly WebClientProvider _webClientProvider; public WebProvider() { _webClientProvider = new WebClientProvider(); _webClientProvider.Timeout = 4; } public bool IgnoreSSL { get => _webClientProvider.IgnoreSSL; set { _webClientProvider.IgnoreSSL = value; } } public async Task RetrieveHtmlDocument(string url) { var sourceCode = await this.RetrieveSourceCode(url); if (string.IsNullOrWhiteSpace(sourceCode)) { return null; } var document = new HtmlAgilityPack.HtmlDocument(); try { document.LoadHtml(sourceCode); } catch (Exception) { return null; } return document; } public async Task RetrieveSourceCode(string url) { if (string.IsNullOrWhiteSpace(url)) { return null; } if (!Uri.TryCreate(url, UriKind.Absolute, out Uri uri)) { return null; } System.Net.Http.HttpResponseMessage response; try { response = await _webClientProvider.Get(url?.Trim()); } catch (Exception) { return null; } if (response == null) { return null; } if (response.StatusCode != HttpStatusCode.OK) { return null; } var sourceCode = ""; try { sourceCode = await response?.Content?.ReadAsStringAsync(); } catch (Exception) { // do nothing } if (string.IsNullOrWhiteSpace(sourceCode)) { return null; } return sourceCode; } public async Task RetrieveImage(string url) { if (string.IsNullOrWhiteSpace(url)) { return null; } if (!Uri.TryCreate(url, UriKind.Absolute, out Uri uri)) { return null; } System.Net.Http.HttpResponseMessage response; try { response = await _webClientProvider.Get(url?.Trim()); } catch (Exception) { return null; } if (response.StatusCode != HttpStatusCode.OK) { return null; } var stream = await response?.Content?.ReadAsStreamAsync(); Image result = null; try { result = Image.FromStream(stream); } catch (Exception) { return null; } return result; } public async Task RetrieveImage(string url, HtmlAgilityPack.HtmlDocument document) { var iconUrl = this.ParseFavicon(document); if (string.IsNullOrWhiteSpace(iconUrl)) { return null; } try { var baseUri = new Uri(url); var absoluteUri = new Uri(baseUri, iconUrl); iconUrl = absoluteUri.AbsoluteUri; } catch { return null; } return await this.RetrieveImage(iconUrl); } public string ParseTitle(HtmlAgilityPack.HtmlDocument document) { string result = null; // Find basic title result = FindNodeValue(document, "//title", string.Empty)?.Trim(); if (!string.IsNullOrWhiteSpace(result)) { return result; } // Find title from extended meta var patternList = new List() { "//meta[@property='og:title']", "//meta[@property='og:site_name']", "//meta[@name='twitter:title']", "//meta[@itemprop='name']" }; foreach (var item in patternList) { result = FindNodeAttrValue(document, item, "content", string.Empty)?.Trim(); if (string.IsNullOrWhiteSpace(result)) { continue; } return result; } return string.Empty; } public string ParseDescription(HtmlAgilityPack.HtmlDocument document) { var patternList = new List() { "//meta[@name='description']", "//meta[@property='og:description']", "//meta[@name='twitter:description']", "//meta[@itemprop='description']", }; foreach (var item in patternList) { var result = FindNodeAttrValue(document, item, "content", string.Empty)?.Trim(); if (string.IsNullOrWhiteSpace(result)) { continue; } return result; } return string.Empty; } public string ParseFavicon(HtmlAgilityPack.HtmlDocument document) { string result = null; // Find link-rel that contains word result = FindNodeAttrValue_ContainsWord(document, "link", "rel", "href", "icon"); if (!string.IsNullOrWhiteSpace(result)) { return result; } // Find link-rel contains apple-icon var matchPatterns = new List() { "apple-touch-icon", "apple-touch-icon-precomposed" }; result = FindNodeAttrValue_Equals(document, "link", "rel", "href", matchPatterns); if (!string.IsNullOrWhiteSpace(result)) { return result; } // Find favicon from extended meta var patternList = new List() { "//meta[@property='og:image']", "//meta[@name='twitter:image']", "//meta[@itemprop='image']" }; foreach (var item in patternList) { result = FindNodeAttrValue(document, item, "content", string.Empty)?.Trim(); if (string.IsNullOrWhiteSpace(result)) { continue; } return result; } return string.Empty; } private string FindNodeValue(HtmlAgilityPack.HtmlDocument document, string xPath, string defaultValue = "") { var hnc = document.DocumentNode.SelectNodes(xPath); if (hnc == null) { return defaultValue; } if (hnc.Count <= 0) { return defaultValue; } foreach (HtmlNode hn in hnc) { if (string.IsNullOrWhiteSpace(hn.InnerHtml)) { continue; } var result = WebUtility.HtmlDecode(hn.InnerHtml)?.Replace("\r", "")?.Replace("\n", " ")?.Trim(); if (string.IsNullOrWhiteSpace(result)) { continue; } return result; } return defaultValue; } private string FindNodeAttrValue(HtmlAgilityPack.HtmlDocument document, string xPath, string attr, string defaultValue = "") { var hnc = document.DocumentNode.SelectNodes(xPath); if (hnc == null) { return defaultValue; } if (hnc.Count <= 0) { return defaultValue; } foreach (HtmlNode hn in hnc) { if (hn.Attributes[attr] == null) { continue; } if (string.IsNullOrWhiteSpace(hn.Attributes[attr].Value)) { continue; } return System.Web.HttpUtility.HtmlDecode(hn.Attributes[attr].Value?.Trim()); } return defaultValue; } private List FindNode(HtmlAgilityPack.HtmlDocument document, string nodeName, string attrName) { var xPath = (string.IsNullOrWhiteSpace(attrName) ? $"//{nodeName}" : $"//{nodeName}[@{attrName}]"); var hnc = document.DocumentNode.SelectNodes(xPath); if (hnc == null) { return new List(); } if (hnc.Count <= 0) { return new List(); } return hnc.ToList(); } private string FindNodeAttrValue_ContainsWord(HtmlAgilityPack.HtmlDocument document, string nodeName, string attrName, string returnAttrName, string matchEqualList, string defaultValue = "") { var linkNodes = FindNode(document, nodeName, attrName); foreach (var item in linkNodes) { var relValue = item.Attributes[attrName].Value?.Trim() ?? string.Empty; if (!ContainsWord(relValue, matchEqualList)) { continue; } var hrefValue = item.Attributes[returnAttrName].Value?.Trim() ?? string.Empty; if (string.IsNullOrWhiteSpace(hrefValue)) { continue; } return System.Web.HttpUtility.HtmlDecode(hrefValue); } return defaultValue; } private string FindNodeAttrValue_Equals(HtmlAgilityPack.HtmlDocument document, string nodeName, string attrName, string returnAttrName, List matchValueList, string defaultValue = "") { var linkNodes = FindNode(document, nodeName, attrName); foreach (var item in linkNodes) { var relValue = item.Attributes[attrName].Value?.Trim() ?? string.Empty; if (!matchValueList.Contains(relValue?.ToLower() ?? string.Empty)) { continue; } var hrefValue = item.Attributes[returnAttrName].Value?.Trim() ?? string.Empty; if (string.IsNullOrWhiteSpace(hrefValue)) { continue; } return System.Web.HttpUtility.HtmlDecode(hrefValue); } return defaultValue; } private bool ContainsWord(string haystack, string needle) { haystack = haystack?.Trim() ?? string.Empty; if (!haystack.Contains(" ")) { return haystack.Equals(needle, StringComparison.CurrentCultureIgnoreCase); } foreach (var item in haystack.Split(" ")) { if (string.IsNullOrWhiteSpace(item)) { continue; } if (item.Equals(needle, StringComparison.CurrentCultureIgnoreCase)) { return true; } } return false; } } }