[Offer] My scraping services.

[Physcadelic]

What the fuck happened to the thread

[Jason]

GetSauce? Gib me some

wordplay <3

@Hassan, I posted my code...MPGH fucked up...hence "MPGH couldn't handle the epicness of my code"

[why06]

Did I miss something?

[Jason]

Did I miss something?

Yah but it's no biggie. task or GTFO now.

[NextGen1]

I let a lot go,but back on topic or I'm changing title to something else.
That's right I said it! , I just may change the title.....!

[Hassan]

I want replica of the information available when we open a thread on MPGH.

Preferable output format: XML

[Jason]

I want replica of the information available when we open a thread on MPGH.

Preferable output format: XML

Okay Hassan, I'll give it a shot

[[MPGH]arunforce]

If you want to do a simple one on (3-4) multiple websites, I have an idea.

Message me on MSN.

[Jason]

If you want to do a simple one on (3-4) multiple websites, I have an idea.

Message me on MSN.

I made VB /

[php]
Imports System.Text.RegularExpressions

Public Class Form1

Private GameList As New List(Of Game)

#Region "Armor"

Public Sub ParseArmorGames()
Dim tags As New List(Of String)(New String() {"Action", "Adventure", "Arcade", "Shooting", "Puzzle-Skill", "Strategy", "Sports", "Miscellaneous"})
For Each s As String In tags
GameList.AddRange(ParseArmorCategory(s))
Next
End Sub

Private Function ParseArmorCategory(ByVal categoryStr As String) As Game()
Dim outList As New List(Of Game)
Dim link As String = "https://armorgames.com/category/" & categoryStr & "/date/1#games"
Dim pageSource As String = GetSauce(link)
Dim mCollect As MatchCollection = Regex.Matches(pageSource, "(?<=<a href=""/category/" & categoryStr & "/date/\d{1,2}(?:#games|/)?"">)\d{1,2}(?=</a>)", RegexOptions.Compiled Or RegexOptions.IgnoreCase)
Dim PageCount As Integer = Integer.Parse(mCollect(mCollec*****unt - 1).Value)
outList.AddRange(ParseArmorPage(pageSource, categoryStr))
If PageCount > 1 Then
For i As Integer = 2 To PageCount
outList.AddRange(ParseArmorPage(GetSauce("https://armorgames.com/category/" & categoryStr & "/date/" & i.ToString & "#games"), categoryStr))
Next
End If
Return outList.ToArray
End Function

Private Function ParseArmorPage(ByVal pageSource As String, ByVal Genre As String) As Game()
Dim outList As New List(Of Game)

Dim first As Integer = pageSource.IndexOf("<div id=""categorylisting"">") + "<div id=""categorylisting"">".Length
Dim last As Integer = pageSource.IndexOf("</ul>", first)
Dim mainPage As String = pageSource.Substring(first, last - first)

Dim gameCollect As MatchCollection = Regex.Matches(mainPage, "(?<=<li>)[\s\S]*?(?=</li>)", RegexOptions.Compiled)

For Each m As Match In gameCollect
Dim gameOverview As String = m.Value
Dim gameInfo As Match = Regex.Match(gameOverview, "<a href=""(?<getLink>.*?)"".*?>[\s\S]*?<h5><a href=""/play/(?<gameNo>\d{1,6})/.*?"" title="".*?"".*?>(?<gameTitle>.*?)</a></h5>", RegexOptions.Compiled)
Dim title As String = gameInfo.Groups("gameTitle").Value
Dim gameNo As String = gameInfo.Groups("gameNo").Value
Dim link As String = "https://armorgames.com/" & gameInfo.Groups("getLink").Value
Dim swfLoc As String = "https://armorgames.com/files/games/" & Web.HttpUtility.HtmlEncode(title.Replace(" "c, "-"c).ToLower) & "-" & gameNo & ".swf"
Dim newGame As New Game(title, link, swfLoc, "Armor Games", Genre)
outList.Add(newGame)
Next
Return outList.ToArray
End Function

#End Region

#Region "Crazy Monkey"

Private Sub ParseCM()
Dim tagList As New List(Of String)(New String() {"Action", "Puzzle", "Arcade", "Fighting", "RPG", "Sport"})
For Each t As String In tagList
GameList.AddRange(ParseCMPage("https://crazymonkeygames.com/tag/" & t.ToLower, t))
Next
End Sub

Private Function ParseCMPage(ByVal tagPage As String, ByVal genre As String) As Game()
Dim outList As New List(Of Game)

Dim src As String = GetSauce(tagPage)
Dim top As String = "<ul class=""search-results"">"
Dim first As Integer = src.IndexOf(top) + top.Length
Dim last As Integer = src.IndexOf("</ul>", first)
Dim mainBlock As String = src.Substring(first, last - first)

Dim gameCollect As MatchCollection = Regex.Matches(mainBlock, "(?<=<li>)[\s\S]*?(?=</li>)", RegexOptions.Compiled)
For Each m As Match In gameCollect
Dim bigMatch As Match = Regex.Match(m.Value, "<h3><a href=""(?<backupLink>.*?)"">(?<gameTitle>.*?)</a>", RegexOptions.Compiled)
Dim title As String = bigMatch.Groups("gameTitle").Value
Dim backupLink As String = bigMatch.Groups("backupLink").Value
Dim predictedSWF As String = "https://www.crazymonkeygames.com/Yt6Rfd/" & title.Replace(" "c, "") & ".swf"
Dim nGame As New Game(title, backupLink, predictedSWF, "Crazy Monkey", genre)
outList.Add(nGame)
Next
Return outList.ToArray
End Function
#End Region

Private Function GetSauce(ByVal URL As String) As String
Try
Dim webReq As Net.HttpWebRequest = CType(Net.WebRequest.Create(URL), Net.HttpWebRequest)
Dim webResp As Net.HttpWebResponse = CType(webReq.GetResponse, Net.HttpWebResponse)
Using sRead As New IO.StreamReader(webResp.GetResponseStream)
Return System.Web.HttpUtility.HtmlDecode(sRead.ReadToEnd)
End Using
Catch ex As Exception
Return "FAIL"
End Try
End Function

End Class

Public Class Game
Private gName As String = String.Empty
Private gLink As String = String.Empty
Private gSwf As String = String.Empty
Private gSite As String = String.Empty
Private gGenre As String = String.Empty

Sub New(ByVal gameTitle As String, ByVal link As String, ByVal gameSwf As String, ByVal gameSite As String, ByVal genre As String)
Me.Name = gameTitle
Me.SwfLocation = gameSwf
Me.Site = gameSite
Me.Genre = genre
Me.Link = link
End Sub

Public Property Site() As String
Get
Return gSite
End Get
Set(ByVal value As String)
gSite = value
End Set
End Property

Public Property Name() As String
Get
Return gName
End Get
Set(ByVal value As String)
gName = value
End Set
End Property

Public Property SwfLocation() As String
Get
Return gSwf
End Get
Set(ByVal value As String)
gSwf = value
End Set
End Property

Public Property Link() As String
Get
Return gLink
End Get
Set(ByVal value As String)
gLink = value
End Set
End Property

Public Property Genre() As String
Get
Return gGenre
End Get
Set(ByVal value As String)
gGenre = value
End Set
End Property
End Class
[/php]

Enjoy

Would be easy enough to add to a database. If you need any help, just hit me up on MSN.

Oh yeah, to parse each one:

[php]
ParseCM()
ParseArmor()
[/php]

And that's it. Simple as.

[Hassan]

Good Job Jason. And since when Arun started VB.NET ?

[why06]

Gawd that's useful! o_O

[Hell_Demon]

I gots to scrape chewing gum from tables at school tomorrow, but since you offer your scraping services ill gladly send you instead! You'll have to get your own plane ticket to holland tho, i'll pick up up and drop u at school.

[Lolland]

You'll have to teach him Hollandaise though, he doesn't speak it fluently.

[why06]

You'll have to teach him Hollandaise though

After you finish send me some, I could use it on my omelet.

[Jason]

You'll have to teach him Hollandaise though, he doesn't speak it fluently.

Stay on topic.

HD, you're fine cos I actually lol'd.

@Hassan, I remember Arun posted something about a VB.NET exam he had to do so