github/mage
1
0
Fork 0
mirror of https://github.com/correl/mage.git synced 2024-12-24 11:50:45 +00:00
Code Issues Projects Releases Packages Wiki Activity

Removed GathererCrawler (moving to stand alone repository)

Browse source
This commit is contained in:
North 2012-07-06 23:49:27 +03:00
parent 9890d3d9c2
commit 22ab84004f
9 changed files with 0 additions and 900 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -36,7 +36,6 @@ Mage.Tests/watchdog.log
Mage/target
Mage.Updater/target
mage.updater.client/target
Utils/GathererCrawler/target
releases
Utils/author.txt

25
Utils/GathererCrawler/pom.xml
View file

@ -1,25 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.mage</groupId>
<artifactId>GathererCrawler</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
<name>GathererCrawler</name>
<url>http://maven.apache.org</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.5.2</version>
</dependency>
</dependencies>
</project>

216
Utils/GathererCrawler/src/main/java/north/gatherercrawler/Card.java
View file

@ -1,216 +0,0 @@
package north.gatherercrawler;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
*
* @author North
*/
public class Card implements Comparable<Card> {
private Integer multiverseId;
private String name;
private List<String> manaCost;
private Integer convertedManaCost;
private String types;
private List<String> cardText;
private List<String> flavorText;
private String powerToughness;
private String expansion;
private String rarity;
private String cardNumber;
private String artist;
private Card otherSide;
public Card(Integer multiverseId) {
this.multiverseId = multiverseId;
}
public Card(String card) {
String[] split = card.split("\\|", 13);
if (split[0].length() > 0) {
multiverseId = Integer.parseInt(split[0]);
}
if (split[1].length() > 0) {
name = split[1];
}
manaCost = new ArrayList<String>();
manaCost.addAll(Arrays.asList(split[2].split("\\$")));
if (split[3].length() > 0) {
convertedManaCost = Integer.parseInt(split[3]);
}
if (split[4].length() > 0) {
types = split[4];
}
cardText = new ArrayList<String>();
cardText.addAll(Arrays.asList(split[5].split("\\$")));
flavorText = new ArrayList<String>();
flavorText.addAll(Arrays.asList(split[6].split("\\$")));
if (split[7].length() > 0) {
powerToughness = split[7];
}
if (split[8].length() > 0) {
expansion = split[8];
}
if (split[9].length() > 0) {
rarity = split[9];
}
if (split[10].length() > 0) {
cardNumber = split[10];
}
if (split[11].length() > 0) {
artist = split[11];
}
}
public String getArtist() {
return artist;
}
public void setArtist(String artist) {
this.artist = artist;
}
public String getCardNumber() {
return cardNumber;
}
public void setCardNumber(String cardNumber) {
this.cardNumber = cardNumber;
}
public List<String> getCardText() {
return cardText;
}
public void setCardText(List<String> cardText) {
this.cardText = cardText;
}
public Integer getConvertedManaCost() {
return convertedManaCost;
}
public void setConvertedManaCost(Integer convertedManaCost) {
this.convertedManaCost = convertedManaCost;
}
public String getExpansion() {
return expansion;
}
public void setExpansion(String expansion) {
this.expansion = expansion;
}
public List<String> getFlavorText() {
return flavorText;
}
public void setFlavorText(List<String> flavorText) {
this.flavorText = flavorText;
}
public List<String> getManaCost() {
return manaCost;
}
public void setManaCost(List<String> manaCost) {
this.manaCost = manaCost;
}
public Integer getMultiverseId() {
return multiverseId;
}
public void setMultiverseId(Integer multiverseId) {
this.multiverseId = multiverseId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getPowerToughness() {
return powerToughness;
}
public void setPowerToughness(String powerToughness) {
this.powerToughness = powerToughness;
}
public String getRarity() {
return rarity;
}
public void setRarity(String rarity) {
this.rarity = rarity;
}
public String getTypes() {
return types;
}
public void setTypes(String types) {
this.types = types;
}
public Card getOtherSide() {
return otherSide;
}
public void setOtherSide(Card otherSide) {
this.otherSide = otherSide;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(multiverseId).append("|");
sb.append(name).append("|");
for (int i = 0; i < manaCost.size(); i++) {
sb.append(manaCost.get(i));
if (i < manaCost.size() - 1) {
sb.append("$");
}
}
sb.append("|");
sb.append(convertedManaCost != null ? convertedManaCost : "").append("|");
sb.append(types).append("|");
for (int i = 0; i < cardText.size(); i++) {
sb.append(cardText.get(i));
if (i < cardText.size() - 1) {
sb.append("$");
}
}
sb.append("|");
for (int i = 0; i < flavorText.size(); i++) {
sb.append(flavorText.get(i));
if (i < flavorText.size() - 1) {
sb.append("$");
}
}
sb.append("|");
sb.append(powerToughness != null ? powerToughness : "").append("|");
sb.append(expansion).append("|");
sb.append(rarity != null ? rarity : "").append("|");
sb.append(cardNumber != null ? cardNumber : "").append("|");
sb.append(artist != null ? artist : "");
if (otherSide != null) {
sb.append("\n").append(otherSide.toString());
}
return sb.toString();
}
public int compareTo(Card o) {
int idCompareResult = this.multiverseId.compareTo(o.getMultiverseId());
return idCompareResult == 0 ? this.cardNumber.compareTo(o.getCardNumber()) : idCompareResult;
}
}

246
Utils/GathererCrawler/src/main/java/north/gatherercrawler/CardParser.java
View file

@ -1,246 +0,0 @@
package north.gatherercrawler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import north.gatherercrawler.util.CardsList;
import north.gatherercrawler.util.ParseQueue;
import north.gatherercrawler.util.ParsedList;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
*
* @author North
*/
public class CardParser extends Thread {
private static final Pattern patternPrint = Pattern.compile("(?<=#)[\\w\\d]+?(?= )");
private static final Pattern patternUrl = Pattern.compile("(?<=/)[\\w\\d]+?(?=\\.html)");
private boolean parseCard(Integer multiverseId) {
String url = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId;
Card card;
Document doc = null;
boolean done = false;
while (!done) {
try {
Connection connection = Jsoup.connect(url);
connection.timeout(20000);
doc = connection.get();
} catch (IOException ex) {
}
done = true;
}
if (!done) {
System.out.println("Card get exception: " + multiverseId);
return false;
}
try {
Elements select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_nameRow .value");
if (!select.isEmpty()) {
card = extractCardData(doc, "", multiverseId);
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_rightCol ul li a");
// for multi-part cards
if (!select.isEmpty()) {
String href = select.attr("href");
url = "http://gatherer.wizards.com/Pages/Card/Details.aspx" + href.substring(href.indexOf("?"));
done = false;
while (!done) {
try {
Connection connection = Jsoup.connect(url);
connection.timeout(20000);
doc = connection.get();
} catch (IOException ex) {
}
done = true;
}
if (!done) {
System.out.println("Card get exception: " + multiverseId);
} else {
card.setCardNumber(card.getCardNumber() + "b");
Card cardSide = extractCardData(doc, "", multiverseId);
cardSide.setCardNumber(cardSide.getCardNumber() + "a");
cardSide.setOtherSide(card);
card = cardSide;
}
}
} else {
// for flip / double sided cards
card = extractCardData(doc, "_ctl05", multiverseId);
if (card == null) {
return false;
}
card.setOtherSide(extractCardData(doc, "_ctl06", multiverseId));
if (card.getOtherSide() == null) {
return false;
}
}
} catch (Exception e) {
return false;
}
if (card == null) {
return false;
}
CardsList.add(card);
return true;
}
private Card extractCardData(Document doc, String selectorModifier, Integer id) throws NumberFormatException {
Elements select;
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_cardImage");
Integer multiverseId = null;
if (!select.isEmpty()) {
Pattern pattern = Pattern.compile("(?<=multiverseid=)\\d+");
Matcher matcher = pattern.matcher(select.get(0).attr("src"));
if (matcher.find()) {
multiverseId = Integer.parseInt(matcher.group());
}
}
if (multiverseId == null) {
return null;
}
Card card = new Card(multiverseId);
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_nameRow .value");
if (!select.isEmpty()) {
card.setName(select.get(0).text().trim());
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_manaRow .value img");
List<String> manaCost = new ArrayList<String>();
if (!select.isEmpty()) {
for (Element element : select) {
manaCost.add(element.attr("src").replace("/Handlers/Image.ashx?size=medium&name=", "").replace("&type=symbol", "").replaceAll("\" alt=\"[\\d\\w\\s]+?\" align=\"absbottom\" />", ""));
}
}
card.setManaCost(manaCost);
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_cmcRow .value");
if (!select.isEmpty()) {
card.setConvertedManaCost(Integer.parseInt(select.get(0).text().trim()));
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_typeRow .value");
if (!select.isEmpty()) {
card.setTypes(select.get(0).text().trim());
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_textRow .value .cardtextbox");
List<String> cardText = new ArrayList<String>();
if (!select.isEmpty()) {
for (Element element : select) {
cardText.add(element.html().trim().replace("<img src=\"/Handlers/Image.ashx?size=small&amp;name=", "{").replace("&amp;type=symbol", "}").replaceAll("\" alt=\"[\\d\\w\\s]+?\" align=\"absbottom\" />", "").replace("\n", "").replace("&quot;", "\""));
}
}
card.setCardText(cardText);
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_FlavorText .cardtextbox");
List<String> flavorText = new ArrayList<String>();
if (!select.isEmpty()) {
for (Element element : select) {
flavorText.add(element.html().trim().replace("&quot;", "\"").replace("<i>", "").replace("</i>", ""));
}
}
card.setFlavorText(flavorText);
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_ptRow .value");
if (!select.isEmpty()) {
card.setPowerToughness(select.get(0).text().trim());
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_currentSetSymbol a");
if (!select.isEmpty()) {
card.setExpansion(select.get(1).text().trim());
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_rarityRow .value span");
if (!select.isEmpty()) {
card.setRarity(select.get(0).text().trim());
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_otherSetsValue a");
List<Integer> otherSets = new ArrayList<Integer>();
if (!select.isEmpty()) {
for (Element element : select) {
otherSets.add(Integer.parseInt(element.attr("href").replace("Details.aspx?multiverseid=", "")));
}
}
// card.setOtherSets(otherSets);
for (Integer otherSet : otherSets) {
if (!ParsedList.contains(otherSet)) {
ParseQueue.add(otherSet);
}
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_numberRow .value");
if (!select.isEmpty()) {
card.setCardNumber(select.get(0).text().trim());
}
select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_ArtistCredit a");
if (!select.isEmpty()) {
card.setArtist(select.get(0).text().trim());
}
if (card.getCardNumber() == null) {
String url = "http://magiccards.info/query?q=" + card.getName().replace(' ', '+');
try {
Connection connection = Jsoup.connect(url);
connection.timeout(20000);
doc = connection.get();
select = doc.select("small a:contains(" + card.getExpansion() + ")");
if (!select.isEmpty()) {
Matcher matcher = patternUrl.matcher(select.get(0).attr("href"));
matcher.find();
card.setCardNumber(matcher.group());
} else {
select = doc.select("small b:contains(#)");
if (!select.isEmpty()) {
Matcher matcher = patternPrint.matcher(select.get(0).html());
matcher.find();
card.setCardNumber(matcher.group());
}
}
} catch (IOException ex) {
}
}
if (card.getCardNumber() == null) {
select = doc.select("p a:contains(" + card.getExpansion() + ")");
if (!select.isEmpty()) {
Matcher matcher = patternUrl.matcher(select.get(0).attr("href"));
matcher.find();
card.setCardNumber(matcher.group());
} else {
select = doc.select("p b:contains(#)");
if (!select.isEmpty()) {
Matcher matcher = patternPrint.matcher(select.get(0).html());
matcher.find();
card.setCardNumber(matcher.group());
}
}
if (card.getCardNumber() == null) {
System.out.println("Card number missing: " + card.getName());
}
}
return card;
}
@Override
public void run() {
while (!ParseQueue.isEmpty()) {
Integer multiverseId = ParseQueue.remove();
if (!ParsedList.contains(multiverseId)) {
ParsedList.add(multiverseId);
parseCard(multiverseId);
}
}
ThreadStarter.threadDone();
}
}

102
Utils/GathererCrawler/src/main/java/north/gatherercrawler/Main.java
View file

@ -1,102 +0,0 @@
package north.gatherercrawler;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
import north.gatherercrawler.util.CardsList;
import north.gatherercrawler.util.ParseQueue;
import north.gatherercrawler.util.ParsedList;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
*
* @author North
*/
public class Main {
private static void readCardsFromFile() {
try {
// Open the file
FileInputStream fstream = new FileInputStream("cards-data.txt");
// Get the object of DataInputStream
DataInputStream in = new DataInputStream(fstream);
BufferedReader br = new BufferedReader(new InputStreamReader(in));
String strLine;
//Read File Line By Line
while ((strLine = br.readLine()) != null) {
if (strLine.length() > 0) {
Card card = new Card(strLine);
CardsList.add(card);
ParsedList.add(card.getMultiverseId());
}
}
//Close the input stream
in.close();
} catch (Exception e) {//Catch exception if any
System.err.println("Error: " + e.getMessage());
}
}
public static void main(String[] args) throws IOException, InterruptedException {
List<String> sets = new ArrayList<String>();
// Change to false if you only want to add a specific set
if (true) {
Document doc = Jsoup.connect("http://gatherer.wizards.com/Pages/Advanced.aspx").get();
Elements select = doc.select("#autoCompleteSourceBoxsetAddText0_InnerTextBoxcontainer a");
if (!select.isEmpty()) {
for (Element element : select) {
if (!element.text().equals("Unglued") && !element.text().equals("Unhinged")) {
sets.add(element.text());
}
}
}
}
readCardsFromFile();
StringBuilder sb = new StringBuilder();
int added = 0;
for (String set : sets) {
sb.append("|[\"").append(set.replace(" ", "+")).append("\"]");
added++;
if (added % 20 == 0 || added == sets.size()) {
int retries = 30;
boolean done = false;
while (retries > 0 && !done) {
String url = "http://gatherer.wizards.com/Pages/Search/Default.aspx?action=advanced&output=checklist&set=" + sb.toString();
Connection connection = Jsoup.connect(url);
connection.timeout(300000);
Document doc = connection.get();
System.out.println(url);
Elements select = doc.select(".checklist .name a");
if (!select.isEmpty()) {
for (Element element : select) {
Integer multiverseId = Integer.parseInt(element.attr("href").replace("../Card/Details.aspx?multiverseid=", ""));
if (!ParsedList.contains(multiverseId)) {
ParseQueue.add(multiverseId);
}
}
}
done = true;
}
if (!done) {
System.out.println("Error accured");
}
sb = new StringBuilder();
Thread.sleep(1000);
}
}
Thread t = new ThreadStarter();
t.start();
}
}

229
Utils/GathererCrawler/src/main/java/north/gatherercrawler/ThreadStarter.java
View file

@ -1,229 +0,0 @@
package north.gatherercrawler;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.util.*;
import north.gatherercrawler.util.CardsList;
/**
*
* @author North
*/
public class ThreadStarter extends Thread {
private static Integer threadsDone = 0;
private final Integer threads = 10;
private List<Card> sortedCards;
public static synchronized void threadDone() {
threadsDone++;
}
private void updateSortedCards() {
if (sortedCards == null) {
sortedCards = new ArrayList<Card>();
Iterator<Card> iterator = CardsList.iterator();
while (iterator.hasNext()) {
sortedCards.add(iterator.next());
}
Collections.sort(sortedCards, new Comparator<Card>() {
public int compare(Card o1, Card o2) {
int expansionCompare = o1.getExpansion().compareTo(o2.getExpansion());
return expansionCompare != 0 ? expansionCompare : o1.getCardNumber().compareTo(o2.getCardNumber());
}
});
}
}
private void writeCardsToFile() {
try {
FileWriter fstream = new FileWriter("cards-data.txt");
BufferedWriter out = new BufferedWriter(fstream);
Iterator<Card> iterator = sortedCards.iterator();
while (iterator.hasNext()) {
out.write(iterator.next().toString());
out.newLine();
}
out.close();
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
}
}
private void writeCardsToUtilFile() {
try {
FileWriter fstream = new FileWriter("mtg-cards-data.txt");
BufferedWriter out = new BufferedWriter(fstream);
Iterator<Card> iterator = sortedCards.iterator();
while (iterator.hasNext()) {
Card card = iterator.next();
String cardLine = generateUtilLine(card);
out.write(replaceSpecialChards(cardLine));
out.newLine();
if (card.getOtherSide() != null) {
cardLine = generateUtilLine(card.getOtherSide());
out.write(replaceSpecialChards(cardLine));
out.newLine();
}
}
out.close();
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
}
}
private void writeCardsToTrackerFile() {
Set<String> knownSets = new TreeSet<String>();
knownSets.add("Fifth Edition");
knownSets.add("Eighth Edition");
knownSets.add("Ninth Edition");
knownSets.add("Tenth Edition");
knownSets.add("Magic 2010");
knownSets.add("Magic 2011");
knownSets.add("Magic 2012");
knownSets.add("Planechase");
knownSets.add("Duel Decks: Elspeth vs. Tezzeret");
knownSets.add("Tempest");
knownSets.add("Urza's Saga");
knownSets.add("Urza's Legacy");
knownSets.add("Invasion");
knownSets.add("Planeshift");
knownSets.add("Apocalypse");
knownSets.add("Onslaught");
knownSets.add("Mirrodin");
knownSets.add("Darksteel");
knownSets.add("Fifth Dawn");
knownSets.add("Champions of Kamigawa");
knownSets.add("Betrayers of Kamigawa");
knownSets.add("Saviors of Kamigawa");
knownSets.add("Ravnica: City of Guilds");
knownSets.add("Guildpact");
knownSets.add("Dissension");
knownSets.add("Time Spiral");
knownSets.add("Time Spiral \"Timeshifted\"");
knownSets.add("Planar Chaos");
knownSets.add("Future Sight");
knownSets.add("Lorwyn");
knownSets.add("Morningtide");
knownSets.add("Shadowmoor");
knownSets.add("Eventide");
knownSets.add("Shards of Alara");
knownSets.add("Conflux");
knownSets.add("Alara Reborn");
knownSets.add("Zendikar");
knownSets.add("Worldwake");
knownSets.add("Rise of the Eldrazi");
knownSets.add("Scars of Mirrodin");
knownSets.add("Mirrodin Besieged");
knownSets.add("New Phyrexia");
knownSets.add("Innistrad");
knownSets.add("Dark Ascension");
knownSets.add("Avacyn Restored");
try {
FileWriter fstream = new FileWriter("mtg-cards-tracker-data.txt");
BufferedWriter out = new BufferedWriter(fstream);
Iterator<Card> iterator = sortedCards.iterator();
while (iterator.hasNext()) {
Card card = iterator.next();
if (knownSets.contains(card.getExpansion())) {
String cardLine = card.toString();
out.write(replaceSpecialChards(cardLine));
out.newLine();
if (card.getOtherSide() != null) {
cardLine = card.getOtherSide().toString();
out.write(replaceSpecialChards(cardLine));
out.newLine();
}
}
}
out.close();
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
}
}
private String replaceSpecialChards(String original) {
return original.replace("\u2014", "-").replace("\u00E9", "e")
.replace("\u00C6", "AE").replace("\u00E6", "ae")
.replace("\u00C1", "A").replace("\u00E1", "a")
.replace("\u00C2", "A").replace("\u00E2", "a")
.replace("\u00D6", "O").replace("\u00F6", "o")
.replace("\u00DB", "U").replace("\u00FB", "u")
.replace("\u00DC", "U").replace("\u00FC", "u");
}
private String generateUtilLine(Card card) {
StringBuilder sb = new StringBuilder();
sb.append(card.getName()).append("|");
sb.append(card.getExpansion()).append("|");
sb.append(card.getCardNumber() != null ? card.getCardNumber() : "").append("|");
String rarity = card.getRarity() != null ? card.getRarity() : "";
if (rarity.equalsIgnoreCase("Mythic Rare")) {
rarity = "M";
}
if (rarity.equalsIgnoreCase("Rare")) {
rarity = "R";
}
if (rarity.equalsIgnoreCase("Uncommon")) {
rarity = "U";
}
if (rarity.equalsIgnoreCase("Common")) {
rarity = "C";
}
if (rarity.equalsIgnoreCase("Basic Land")) {
rarity = "L";
}
sb.append(rarity).append("|");
List<String> manaCost = card.getManaCost();
for (String cost : manaCost) {
if (!cost.isEmpty()) {
sb.append("{").append(cost).append("}");
}
}
sb.append("|");
sb.append(card.getTypes()).append("|");
String pts = card.getPowerToughness();
if (pts != null && pts.length() > 1) {
String[] pt = pts.split("/");
sb.append(pt[0].trim()).append("|");
sb.append(pt[1].trim()).append("|");
} else {
sb.append("||");
}
List<String> cardText = card.getCardText();
for (int i = 0; i < cardText.size(); i++) {
sb.append(cardText.get(i));
if (i < cardText.size() - 1) {
sb.append("$");
}
}
sb.append("|");
return sb.toString();
}
@Override
public void run() {
for (int i = 0; i < threads; i++) {
Thread t = new CardParser();
t.start();
}
while (threads != threadsDone) {
try {
synchronized (this) {
this.wait(5000);
}
} catch (InterruptedException ex) {
}
}
updateSortedCards();
writeCardsToFile();
writeCardsToUtilFile();
writeCardsToTrackerFile();
}
}

27
Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/CardsList.java
View file

@ -1,27 +0,0 @@
package north.gatherercrawler.util;
import java.util.Iterator;
import java.util.concurrent.ConcurrentSkipListSet;
import north.gatherercrawler.Card;
/**
*
* @author North
*/
public class CardsList {
private static final CardsList instance = new CardsList();
private ConcurrentSkipListSet<Card> list;
public CardsList() {
list = new ConcurrentSkipListSet<Card>();
}
public static void add(Card element) {
instance.list.add(element);
}
public static Iterator<Card> iterator() {
return instance.list.iterator();
}
}

29
Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParseQueue.java
View file

@ -1,29 +0,0 @@
package north.gatherercrawler.util;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
*
* @author North
*/
public class ParseQueue {
private static final ParseQueue instance = new ParseQueue();
private ConcurrentLinkedQueue<Integer> queue;
public ParseQueue() {
queue = new ConcurrentLinkedQueue<Integer>();
}
public static void add(Integer element) {
instance.queue.add(element);
}
public static Integer remove() {
return instance.queue.remove();
}
public static boolean isEmpty(){
return instance.queue.isEmpty();
}
}

25
Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParsedList.java
View file

@ -1,25 +0,0 @@
package north.gatherercrawler.util;
import java.util.concurrent.ConcurrentSkipListSet;
/**
*
* @author North
*/
public class ParsedList {
private static final ParsedList instance = new ParsedList();
private ConcurrentSkipListSet<Integer> list;
public ParsedList() {
list = new ConcurrentSkipListSet<Integer>();
}
public static void add(Integer element) {
instance.list.add(element);
}
public static boolean contains(Integer value) {
return instance.list.contains(value);
}
}