bookmark-manager-r4/Services/WebProvider.cs

using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using HtmlAgilityPack;
using RyzStudio.Net;

namespace BookmarkManager.Services
{
    public class WebProvider
    {
        private readonly WebClientProvider _webClientProvider;


        public WebProvider()
        {
            _webClientProvider = new WebClientProvider();
            _webClientProvider.Timeout = 4;
        }


        public bool IgnoreSSL
        {
            get => _webClientProvider.IgnoreSSL;
            set
            {
                _webClientProvider.IgnoreSSL = value;
            }
        }


        public async Task<HtmlAgilityPack.HtmlDocument> RetrieveHtmlDocument(string url)
        {
            var sourceCode = await this.RetrieveSourceCode(url);
            if (string.IsNullOrWhiteSpace(sourceCode))
            {
                return null;
            }

            var document = new HtmlAgilityPack.HtmlDocument();

            try
            {
                document.LoadHtml(sourceCode);
            }
            catch (Exception)
            {
                return null;
            }

            return document;
        }

        public async Task<string> RetrieveSourceCode(string url)
        {
            if (string.IsNullOrWhiteSpace(url))
            {
                return null;
            }

            if (!Uri.TryCreate(url, UriKind.Absolute, out Uri uri))
            {
                return null;
            }

            System.Net.Http.HttpResponseMessage response;

            try
            {
                response = await _webClientProvider.Get(url?.Trim());
            }
            catch (Exception)
            {
                return null;
            }

            if (response == null)
            {
                return null;

            }

            if (response.StatusCode != HttpStatusCode.OK)
            {
                return null;
            }

            var sourceCode = "";

            try
            {
                sourceCode = await response?.Content?.ReadAsStringAsync();
            }
            catch (Exception)
            {
                // do nothing
            }

            if (string.IsNullOrWhiteSpace(sourceCode))
            {
                return null;
            }

            return sourceCode;
        }

        public async Task<Image> RetrieveImage(string url)
        {
            if (string.IsNullOrWhiteSpace(url))
            {
                return null;
            }

            if (!Uri.TryCreate(url, UriKind.Absolute, out Uri uri))
            {
                return null;
            }

            System.Net.Http.HttpResponseMessage response;

            try
            {
                response = await _webClientProvider.Get(url?.Trim());
            }
            catch (Exception)
            {
                return null;
            }

            if (response.StatusCode != HttpStatusCode.OK)
            {
                return null;
            }

            var stream = await response?.Content?.ReadAsStreamAsync();

            Image result = null;

            try
            {
                result = Image.FromStream(stream);
            }
            catch (Exception)
            {
                return null;
            }

            return result;
        }

        public async Task<Image> RetrieveImage(string url, HtmlAgilityPack.HtmlDocument document)
        {
            var iconUrl = this.ParseFavicon(document);
            if (string.IsNullOrWhiteSpace(iconUrl))
            {
                return null;
            }

            try
            {
                var baseUri = new Uri(url);
                var absoluteUri = new Uri(baseUri, iconUrl);

                iconUrl = absoluteUri.AbsoluteUri;
            }
            catch
            {
                return null;
            }

            return await this.RetrieveImage(iconUrl);
        }

        public string ParseTitle(HtmlAgilityPack.HtmlDocument document)
        {
            string result = null;

            // Find basic title
            result = FindNodeValue(document, "//title", string.Empty)?.Trim();
            if (!string.IsNullOrWhiteSpace(result))
            {
                return result;
            }

            // Find title from extended meta
            var patternList = new List<string>()
            {
                "//meta[@property='og:title']",
                "//meta[@property='og:site_name']",
                "//meta[@name='twitter:title']",
                "//meta[@itemprop='name']"
            };

            foreach (var item in patternList)
            {
                result = FindNodeAttrValue(document, item, "content", string.Empty)?.Trim();
                if (string.IsNullOrWhiteSpace(result))
                {
                    continue;
                }

                return result;
            }

            return string.Empty;
        }

        public string ParseDescription(HtmlAgilityPack.HtmlDocument document)
        {
            var patternList = new List<string>()
            {
                "//meta[@name='description']",
                "//meta[@property='og:description']",
                "//meta[@name='twitter:description']",
                "//meta[@itemprop='description']",
            };

            foreach (var item in patternList)
            {
                var result = FindNodeAttrValue(document, item, "content", string.Empty)?.Trim();
                if (string.IsNullOrWhiteSpace(result))
                {
                    continue;
                }

                return result;
            }

            return string.Empty;
        }

        public string ParseFavicon(HtmlAgilityPack.HtmlDocument document)
        {
            string result = null;

            // Find link-rel that contains word
            result = FindNodeAttrValue_ContainsWord(document, "link", "rel", "href", "icon");
            if (!string.IsNullOrWhiteSpace(result))
            {
                return result;
            }

            // Find link-rel contains apple-icon
            var matchPatterns = new List<string>() { "apple-touch-icon", "apple-touch-icon-precomposed" };
            result = FindNodeAttrValue_Equals(document, "link", "rel", "href", matchPatterns);
            if (!string.IsNullOrWhiteSpace(result))
            {
                return result;
            }

            // Find favicon from extended meta
            var patternList = new List<string>()
            {
                "//meta[@property='og:image']",
                "//meta[@name='twitter:image']",
                "//meta[@itemprop='image']"
            };

            foreach (var item in patternList)
            {
                result = FindNodeAttrValue(document, item, "content", string.Empty)?.Trim();
                if (string.IsNullOrWhiteSpace(result))
                {
                    continue;
                }

                return result;
            }

            return string.Empty;
        }

        private string FindNodeValue(HtmlAgilityPack.HtmlDocument document, string xPath, string defaultValue = "")
        {
            var hnc = document.DocumentNode.SelectNodes(xPath);
            if (hnc == null)
            {
                return defaultValue;
            }

            if (hnc.Count <= 0)
            {
                return defaultValue;
            }

            foreach (HtmlNode hn in hnc)
            {
                if (string.IsNullOrWhiteSpace(hn.InnerHtml))
                {
                    continue;
                }

                var result = WebUtility.HtmlDecode(hn.InnerHtml)?.Replace("\r", "")?.Replace("\n", " ")?.Trim();
                if (string.IsNullOrWhiteSpace(result))
                {
                    continue;
                }

                return result;
            }

            return defaultValue;
        }

        private string FindNodeAttrValue(HtmlAgilityPack.HtmlDocument document, string xPath, string attr, string defaultValue = "")
        {
            var hnc = document.DocumentNode.SelectNodes(xPath);
            if (hnc == null)
            {
                return defaultValue;
            }

            if (hnc.Count <= 0)
            {
                return defaultValue;
            }

            foreach (HtmlNode hn in hnc)
            {
                if (hn.Attributes[attr] == null)
                {
                    continue;
                }

                if (string.IsNullOrWhiteSpace(hn.Attributes[attr].Value))
                {
                    continue;
                }

                return System.Web.HttpUtility.HtmlDecode(hn.Attributes[attr].Value?.Trim());
            }

            return defaultValue;
        }

        private List<HtmlNode> FindNode(HtmlAgilityPack.HtmlDocument document, string nodeName, string attrName)
        {
            var xPath = (string.IsNullOrWhiteSpace(attrName) ? $"//{nodeName}" : $"//{nodeName}[@{attrName}]");
            var hnc = document.DocumentNode.SelectNodes(xPath);
            if (hnc == null)
            {
                return new List<HtmlNode>();
            }

            if (hnc.Count <= 0)
            {
                return new List<HtmlNode>();
            }

            return hnc.ToList();
        }

        private string FindNodeAttrValue_ContainsWord(HtmlAgilityPack.HtmlDocument document, string nodeName, string attrName, string returnAttrName, string matchEqualList, string defaultValue = "")
        {
            var linkNodes = FindNode(document, nodeName, attrName);
            foreach (var item in linkNodes)
            {
                var relValue = item.Attributes[attrName].Value?.Trim() ?? string.Empty;
                if (!ContainsWord(relValue, matchEqualList))
                {
                    continue;
                }

                var hrefValue = item.Attributes[returnAttrName].Value?.Trim() ?? string.Empty;
                if (string.IsNullOrWhiteSpace(hrefValue))
                {
                    continue;
                }

                return System.Web.HttpUtility.HtmlDecode(hrefValue);
            }

            return defaultValue;
        }

        private string FindNodeAttrValue_Equals(HtmlAgilityPack.HtmlDocument document, string nodeName, string attrName, string returnAttrName, List<string> matchValueList, string defaultValue = "")
        {
            var linkNodes = FindNode(document, nodeName, attrName);
            foreach (var item in linkNodes)
            {
                var relValue = item.Attributes[attrName].Value?.Trim() ?? string.Empty;
                if (!matchValueList.Contains(relValue?.ToLower() ?? string.Empty))
                {
                    continue;
                }

                var hrefValue = item.Attributes[returnAttrName].Value?.Trim() ?? string.Empty;
                if (string.IsNullOrWhiteSpace(hrefValue))
                {
                    continue;
                }

                return System.Web.HttpUtility.HtmlDecode(hrefValue);
            }

            return defaultValue;
        }

        private bool ContainsWord(string haystack, string needle)
        {
            haystack = haystack?.Trim() ?? string.Empty;

            if (!haystack.Contains(" "))
            {
                return haystack.Equals(needle, StringComparison.CurrentCultureIgnoreCase);
            }

            foreach (var item in haystack.Split(" "))
            {
                if (string.IsNullOrWhiteSpace(item))
                {
                    continue;
                }

                if (item.Equals(needle, StringComparison.CurrentCultureIgnoreCase))
                {
                    return true;
                }
            }

            return false;
        }

    }
}