SolvedRunning multiple threads

using System;
using System.Threading;
using System.Threading.Tasks;
using System.Collections.Generic;

namespace JLibrary.Net {
    public class SparseScheduler : TaskScheduler {
        [ThreadStatic]
        private static bool _currentThreadIsProcessingItems;

        private readonly LinkedList<Task> _waitingTasks = new LinkedList<Task>();
        private readonly int _maxConcurrent;
        private int _runningDelegates;

        public SparseScheduler(int maximumConcurrent) {
            if (maximumConcurrent < 1) {
                throw new ArgumentOutOfRangeException("maximumConcurrent");
            }

            _maxConcurrent = maximumConcurrent;
            _runningDelegates = 0;
        }

        protected override void QueueTask(Task task) {
            lock (_waitingTasks) {
                _waitingTasks.AddLast(task);
                if (_runningDelegates < _maxConcurrent) {
                    ++_runningDelegates;
                    NotifyThreadPool();
                }
            }
        }

        private void NotifyThreadPool() {
            ThreadPool.UnsafeQueueUserWorkItem(_ => {
                _currentThreadIsProcessingItems = true;
                try {
                    while (true) {
                        Task item;
                        lock (_waitingTasks) {
                            if (_waitingTasks.Count == 0) {
                                --_runningDelegates;
                                break;
                            }

                            item = _waitingTasks.First.Value;
                            _waitingTasks.RemoveFirst();
                        }

                        base.TryExecuteTask(item);
                    }
                }
                finally {
                    _currentThreadIsProcessingItems = false;
                }
            }, null);
        }

        protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) {
            if (!_currentThreadIsProcessingItems) {
                return false;
            }

            return taskWasPreviouslyQueued
                ? TryDequeue(task) && base.TryExecuteTask(task)
                : base.TryExecuteTask(task);
        }

        protected override bool TryDequeue(Task task) {
            lock (_waitingTasks) {
                return _waitingTasks.Remove(task);
            }
        }

        protected override IEnumerable<Task> GetScheduledTasks() {
            bool lockTaken = false;
            try {
                Monitor.TryEnter(_waitingTasks, ref lockTaken);
                if (lockTaken) return _waitingTasks;
                else throw new NotSupportedException();
            }
            finally {
                if (lockTaken) {
                    Monitor.Exit(_waitingTasks);
                }
            }
        }
    }
}

using System;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using JLibrary.Threading;

namespace JLibrary.Net {
    // Weak wrapper around a Task
    public class WebRequestToken {
        private readonly Task _task;
        internal Task Task { get { return _task; } }

        internal WebRequestToken(Task task) {
            _task = task;
        }

        private WebRequestToken() {
            // Hide the empty constructor
        }
    };

    public class WebRequestFactory {
        private readonly TaskFactory _factory;
        private readonly SparseScheduler _scheduler;
        private readonly CancellationTokenSource _cts;

        public WebRequestFactory(int maxConcurrentRequests) {
            _scheduler = new SparseScheduler(maxConcurrentRequests);
            _factory = new TaskFactory(_scheduler);
            _cts = new CancellationTokenSource();
        }

        public WebRequestToken SendRequest(WebRequest request, Action<Exception,WebResponse> handler) {
            if (request == null)
                throw new ArgumentNullException("request");
            if (handler == null)
                throw new ArgumentNullException("handler");

            var task = _factory.StartNew(() => {
                try {
                    using (var response = request.GetResponse())
                        handler.Invoke(null, response);
                }
                catch (Exception e) {
                    handler.Invoke(e, null);
                }
            }, _cts.Token);

            return new WebRequestToken(task);
        }

        public void WaitAll(params WebRequestToken[] tokens) {
            Task.WaitAll(Array.ConvertAll(tokens, t => t.Task), _cts.Token);
        }

        public void WaitAll(WebRequestToken[] tokens, int millisecondsToWait) {
            Task.WaitAll(Array.ConvertAll(tokens, t => t.Task), millisecondsToWait, _cts.Token);
        }
    }
}

using JLibrary.Net;

private void HandleResponse(Exception e, WebResponse response) {
    if (e != null) {
        // do something with the exception
        throw e;
    }

    // Store a local "searched" counter to avoid locking the mutex every time a link is found
    int searched = 0;
    using(var sreader = new StreamReader(response.GetResponseStream())) {
        for(string line = sreader.ReadLine(); line != null; line = sreader.ReadLine()) {
            if (line.Contains("title story-title")) {
                ++searched;
                var title = line.Replace("<h1 class=\"title story-title\">", "").Replace("</h1>", "").Trim();
                loglink(response.ResponseUri.ToString(), title);
            }
        }
    }

    lock(_mutex) {
        Links_Searched += searched;
    }
}

private readonly object _mutex = new object();

int maxConcurrent = 8; // Maximum 8 web requests running at any time.
int increment = (int)IncrementNum.Value;

var factory = new WebRequestFactory(maxConcurrent);
var tokens = new List<WebRequestToken>();

for(Search_CurrentID+=increment; Search_CurrentID < Search_MaxID; Search_CurrentID+=increment) {
    var link = string.Format("{0}{1}", link_mask, Search_CurrentID);
    var req = (HttpWebRequest)WebRequest.Create(link);
    req.Timeout = 2000;
    req.Method = WebRequestMethods.Http.Post;
    req.ContentType = "application/x-www-form-urlencoded";
    req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64_R3D) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36";
    req.ContentLength = 0;

    tokens.Add(factory.SendRequest(req, HandleResponse));
}

// Now wait for the requests to complete
factory.WaitAll(tokens.ToArray());

// Done.