From 22ab84004f038594f8a6ea5cac1675e2a597437a Mon Sep 17 00:00:00 2001 From: North Date: Fri, 6 Jul 2012 23:49:27 +0300 Subject: [PATCH] Removed GathererCrawler (moving to stand alone repository) --- .gitignore | 1 - Utils/GathererCrawler/pom.xml | 25 -- .../main/java/north/gatherercrawler/Card.java | 216 --------------- .../north/gatherercrawler/CardParser.java | 246 ------------------ .../main/java/north/gatherercrawler/Main.java | 102 -------- .../north/gatherercrawler/ThreadStarter.java | 229 ---------------- .../north/gatherercrawler/util/CardsList.java | 27 -- .../gatherercrawler/util/ParseQueue.java | 29 --- .../gatherercrawler/util/ParsedList.java | 25 -- 9 files changed, 900 deletions(-) delete mode 100644 Utils/GathererCrawler/pom.xml delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/Card.java delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/CardParser.java delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/Main.java delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/ThreadStarter.java delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/CardsList.java delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParseQueue.java delete mode 100644 Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParsedList.java diff --git a/.gitignore b/.gitignore index 5c64cf0b45..af311d29c0 100644 --- a/.gitignore +++ b/.gitignore @@ -36,7 +36,6 @@ Mage.Tests/watchdog.log Mage/target Mage.Updater/target mage.updater.client/target -Utils/GathererCrawler/target releases Utils/author.txt diff --git a/Utils/GathererCrawler/pom.xml b/Utils/GathererCrawler/pom.xml deleted file mode 100644 index 197aaa8562..0000000000 --- a/Utils/GathererCrawler/pom.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - 4.0.0 - - org.mage - GathererCrawler - 1.0 - jar - - GathererCrawler - http://maven.apache.org - - - UTF-8 - - - - - org.jsoup - jsoup - 1.5.2 - - - diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/Card.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/Card.java deleted file mode 100644 index 0519a41d22..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/Card.java +++ /dev/null @@ -1,216 +0,0 @@ -package north.gatherercrawler; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -/** - * - * @author North - */ -public class Card implements Comparable { - - private Integer multiverseId; - private String name; - private List manaCost; - private Integer convertedManaCost; - private String types; - private List cardText; - private List flavorText; - private String powerToughness; - private String expansion; - private String rarity; - private String cardNumber; - private String artist; - private Card otherSide; - - public Card(Integer multiverseId) { - this.multiverseId = multiverseId; - } - - public Card(String card) { - String[] split = card.split("\\|", 13); - if (split[0].length() > 0) { - multiverseId = Integer.parseInt(split[0]); - } - if (split[1].length() > 0) { - name = split[1]; - } - manaCost = new ArrayList(); - manaCost.addAll(Arrays.asList(split[2].split("\\$"))); - if (split[3].length() > 0) { - convertedManaCost = Integer.parseInt(split[3]); - } - if (split[4].length() > 0) { - types = split[4]; - } - cardText = new ArrayList(); - cardText.addAll(Arrays.asList(split[5].split("\\$"))); - flavorText = new ArrayList(); - flavorText.addAll(Arrays.asList(split[6].split("\\$"))); - if (split[7].length() > 0) { - powerToughness = split[7]; - } - if (split[8].length() > 0) { - expansion = split[8]; - } - if (split[9].length() > 0) { - rarity = split[9]; - } - if (split[10].length() > 0) { - cardNumber = split[10]; - } - if (split[11].length() > 0) { - artist = split[11]; - } - } - - public String getArtist() { - return artist; - } - - public void setArtist(String artist) { - this.artist = artist; - } - - public String getCardNumber() { - return cardNumber; - } - - public void setCardNumber(String cardNumber) { - this.cardNumber = cardNumber; - } - - public List getCardText() { - return cardText; - } - - public void setCardText(List cardText) { - this.cardText = cardText; - } - - public Integer getConvertedManaCost() { - return convertedManaCost; - } - - public void setConvertedManaCost(Integer convertedManaCost) { - this.convertedManaCost = convertedManaCost; - } - - public String getExpansion() { - return expansion; - } - - public void setExpansion(String expansion) { - this.expansion = expansion; - } - - public List getFlavorText() { - return flavorText; - } - - public void setFlavorText(List flavorText) { - this.flavorText = flavorText; - } - - public List getManaCost() { - return manaCost; - } - - public void setManaCost(List manaCost) { - this.manaCost = manaCost; - } - - public Integer getMultiverseId() { - return multiverseId; - } - - public void setMultiverseId(Integer multiverseId) { - this.multiverseId = multiverseId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getPowerToughness() { - return powerToughness; - } - - public void setPowerToughness(String powerToughness) { - this.powerToughness = powerToughness; - } - - public String getRarity() { - return rarity; - } - - public void setRarity(String rarity) { - this.rarity = rarity; - } - - public String getTypes() { - return types; - } - - public void setTypes(String types) { - this.types = types; - } - - public Card getOtherSide() { - return otherSide; - } - - public void setOtherSide(Card otherSide) { - this.otherSide = otherSide; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(multiverseId).append("|"); - sb.append(name).append("|"); - for (int i = 0; i < manaCost.size(); i++) { - sb.append(manaCost.get(i)); - if (i < manaCost.size() - 1) { - sb.append("$"); - } - } - sb.append("|"); - sb.append(convertedManaCost != null ? convertedManaCost : "").append("|"); - sb.append(types).append("|"); - for (int i = 0; i < cardText.size(); i++) { - sb.append(cardText.get(i)); - if (i < cardText.size() - 1) { - sb.append("$"); - } - } - sb.append("|"); - for (int i = 0; i < flavorText.size(); i++) { - sb.append(flavorText.get(i)); - if (i < flavorText.size() - 1) { - sb.append("$"); - } - } - sb.append("|"); - sb.append(powerToughness != null ? powerToughness : "").append("|"); - sb.append(expansion).append("|"); - sb.append(rarity != null ? rarity : "").append("|"); - sb.append(cardNumber != null ? cardNumber : "").append("|"); - sb.append(artist != null ? artist : ""); - - if (otherSide != null) { - sb.append("\n").append(otherSide.toString()); - } - return sb.toString(); - } - - public int compareTo(Card o) { - int idCompareResult = this.multiverseId.compareTo(o.getMultiverseId()); - return idCompareResult == 0 ? this.cardNumber.compareTo(o.getCardNumber()) : idCompareResult; - } -} diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/CardParser.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/CardParser.java deleted file mode 100644 index 448c94745a..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/CardParser.java +++ /dev/null @@ -1,246 +0,0 @@ -package north.gatherercrawler; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import north.gatherercrawler.util.CardsList; -import north.gatherercrawler.util.ParseQueue; -import north.gatherercrawler.util.ParsedList; -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -/** - * - * @author North - */ -public class CardParser extends Thread { - - private static final Pattern patternPrint = Pattern.compile("(?<=#)[\\w\\d]+?(?= )"); - private static final Pattern patternUrl = Pattern.compile("(?<=/)[\\w\\d]+?(?=\\.html)"); - - private boolean parseCard(Integer multiverseId) { - String url = "http://gatherer.wizards.com/Pages/Card/Details.aspx?multiverseid=" + multiverseId; - Card card; - Document doc = null; - boolean done = false; - while (!done) { - try { - Connection connection = Jsoup.connect(url); - connection.timeout(20000); - doc = connection.get(); - } catch (IOException ex) { - } - done = true; - } - if (!done) { - System.out.println("Card get exception: " + multiverseId); - return false; - } - - try { - Elements select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_nameRow .value"); - if (!select.isEmpty()) { - card = extractCardData(doc, "", multiverseId); - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent_rightCol ul li a"); - - // for multi-part cards - if (!select.isEmpty()) { - String href = select.attr("href"); - url = "http://gatherer.wizards.com/Pages/Card/Details.aspx" + href.substring(href.indexOf("?")); - - done = false; - while (!done) { - try { - Connection connection = Jsoup.connect(url); - connection.timeout(20000); - doc = connection.get(); - } catch (IOException ex) { - } - done = true; - } - if (!done) { - System.out.println("Card get exception: " + multiverseId); - } else { - card.setCardNumber(card.getCardNumber() + "b"); - Card cardSide = extractCardData(doc, "", multiverseId); - cardSide.setCardNumber(cardSide.getCardNumber() + "a"); - cardSide.setOtherSide(card); - card = cardSide; - } - } - } else { - // for flip / double sided cards - card = extractCardData(doc, "_ctl05", multiverseId); - if (card == null) { - return false; - } - card.setOtherSide(extractCardData(doc, "_ctl06", multiverseId)); - if (card.getOtherSide() == null) { - return false; - } - } - } catch (Exception e) { - return false; - } - - - if (card == null) { - return false; - } - CardsList.add(card); - return true; - } - - private Card extractCardData(Document doc, String selectorModifier, Integer id) throws NumberFormatException { - Elements select; - - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_cardImage"); - Integer multiverseId = null; - if (!select.isEmpty()) { - Pattern pattern = Pattern.compile("(?<=multiverseid=)\\d+"); - Matcher matcher = pattern.matcher(select.get(0).attr("src")); - if (matcher.find()) { - multiverseId = Integer.parseInt(matcher.group()); - } - } - if (multiverseId == null) { - return null; - } - - Card card = new Card(multiverseId); - - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_nameRow .value"); - if (!select.isEmpty()) { - card.setName(select.get(0).text().trim()); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_manaRow .value img"); - List manaCost = new ArrayList(); - if (!select.isEmpty()) { - for (Element element : select) { - manaCost.add(element.attr("src").replace("/Handlers/Image.ashx?size=medium&name=", "").replace("&type=symbol", "").replaceAll("\" alt=\"[\\d\\w\\s]+?\" align=\"absbottom\" />", "")); - } - } - card.setManaCost(manaCost); - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_cmcRow .value"); - if (!select.isEmpty()) { - card.setConvertedManaCost(Integer.parseInt(select.get(0).text().trim())); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_typeRow .value"); - if (!select.isEmpty()) { - card.setTypes(select.get(0).text().trim()); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_textRow .value .cardtextbox"); - List cardText = new ArrayList(); - if (!select.isEmpty()) { - for (Element element : select) { - cardText.add(element.html().trim().replace("\"[\\d\\w\\s]+?\"", "").replace("\n", "").replace(""", "\"")); - } - } - card.setCardText(cardText); - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_FlavorText .cardtextbox"); - List flavorText = new ArrayList(); - if (!select.isEmpty()) { - for (Element element : select) { - flavorText.add(element.html().trim().replace(""", "\"").replace("", "").replace("", "")); - } - } - card.setFlavorText(flavorText); - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_ptRow .value"); - if (!select.isEmpty()) { - card.setPowerToughness(select.get(0).text().trim()); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_currentSetSymbol a"); - if (!select.isEmpty()) { - card.setExpansion(select.get(1).text().trim()); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_rarityRow .value span"); - if (!select.isEmpty()) { - card.setRarity(select.get(0).text().trim()); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_otherSetsValue a"); - List otherSets = new ArrayList(); - if (!select.isEmpty()) { - for (Element element : select) { - otherSets.add(Integer.parseInt(element.attr("href").replace("Details.aspx?multiverseid=", ""))); - } - } - // card.setOtherSets(otherSets); - for (Integer otherSet : otherSets) { - if (!ParsedList.contains(otherSet)) { - ParseQueue.add(otherSet); - } - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_numberRow .value"); - if (!select.isEmpty()) { - card.setCardNumber(select.get(0).text().trim()); - } - select = doc.select("#ctl00_ctl00_ctl00_MainContent_SubContent_SubContent" + selectorModifier + "_ArtistCredit a"); - if (!select.isEmpty()) { - card.setArtist(select.get(0).text().trim()); - } - - if (card.getCardNumber() == null) { - String url = "http://magiccards.info/query?q=" + card.getName().replace(' ', '+'); - try { - Connection connection = Jsoup.connect(url); - connection.timeout(20000); - doc = connection.get(); - - select = doc.select("small a:contains(" + card.getExpansion() + ")"); - if (!select.isEmpty()) { - Matcher matcher = patternUrl.matcher(select.get(0).attr("href")); - matcher.find(); - card.setCardNumber(matcher.group()); - } else { - select = doc.select("small b:contains(#)"); - if (!select.isEmpty()) { - Matcher matcher = patternPrint.matcher(select.get(0).html()); - matcher.find(); - card.setCardNumber(matcher.group()); - } - } - - } catch (IOException ex) { - } - } - - if (card.getCardNumber() == null) { - select = doc.select("p a:contains(" + card.getExpansion() + ")"); - if (!select.isEmpty()) { - Matcher matcher = patternUrl.matcher(select.get(0).attr("href")); - matcher.find(); - card.setCardNumber(matcher.group()); - } else { - select = doc.select("p b:contains(#)"); - if (!select.isEmpty()) { - Matcher matcher = patternPrint.matcher(select.get(0).html()); - matcher.find(); - card.setCardNumber(matcher.group()); - } - } - if (card.getCardNumber() == null) { - System.out.println("Card number missing: " + card.getName()); - } - } - - return card; - } - - @Override - public void run() { - while (!ParseQueue.isEmpty()) { - Integer multiverseId = ParseQueue.remove(); - if (!ParsedList.contains(multiverseId)) { - ParsedList.add(multiverseId); - parseCard(multiverseId); - } - } - - ThreadStarter.threadDone(); - } -} diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/Main.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/Main.java deleted file mode 100644 index 5f7b516558..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/Main.java +++ /dev/null @@ -1,102 +0,0 @@ -package north.gatherercrawler; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; -import north.gatherercrawler.util.CardsList; -import north.gatherercrawler.util.ParseQueue; -import north.gatherercrawler.util.ParsedList; -import org.jsoup.Connection; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; - -/** - * - * @author North - */ -public class Main { - - private static void readCardsFromFile() { - try { - // Open the file - FileInputStream fstream = new FileInputStream("cards-data.txt"); - // Get the object of DataInputStream - DataInputStream in = new DataInputStream(fstream); - BufferedReader br = new BufferedReader(new InputStreamReader(in)); - String strLine; - //Read File Line By Line - while ((strLine = br.readLine()) != null) { - if (strLine.length() > 0) { - Card card = new Card(strLine); - CardsList.add(card); - ParsedList.add(card.getMultiverseId()); - } - } - //Close the input stream - in.close(); - } catch (Exception e) {//Catch exception if any - System.err.println("Error: " + e.getMessage()); - } - } - - public static void main(String[] args) throws IOException, InterruptedException { - List sets = new ArrayList(); - - // Change to false if you only want to add a specific set - if (true) { - Document doc = Jsoup.connect("http://gatherer.wizards.com/Pages/Advanced.aspx").get(); - Elements select = doc.select("#autoCompleteSourceBoxsetAddText0_InnerTextBoxcontainer a"); - if (!select.isEmpty()) { - for (Element element : select) { - if (!element.text().equals("Unglued") && !element.text().equals("Unhinged")) { - sets.add(element.text()); - } - } - } - } - - readCardsFromFile(); - - StringBuilder sb = new StringBuilder(); - int added = 0; - for (String set : sets) { - sb.append("|[\"").append(set.replace(" ", "+")).append("\"]"); - added++; - - if (added % 20 == 0 || added == sets.size()) { - int retries = 30; - boolean done = false; - while (retries > 0 && !done) { - String url = "http://gatherer.wizards.com/Pages/Search/Default.aspx?action=advanced&output=checklist&set=" + sb.toString(); - Connection connection = Jsoup.connect(url); - connection.timeout(300000); - Document doc = connection.get(); - System.out.println(url); - - Elements select = doc.select(".checklist .name a"); - if (!select.isEmpty()) { - for (Element element : select) { - Integer multiverseId = Integer.parseInt(element.attr("href").replace("../Card/Details.aspx?multiverseid=", "")); - if (!ParsedList.contains(multiverseId)) { - ParseQueue.add(multiverseId); - } - } - } - done = true; - } - - if (!done) { - System.out.println("Error accured"); - } - sb = new StringBuilder(); - Thread.sleep(1000); - } - } - - - Thread t = new ThreadStarter(); - t.start(); - } -} diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/ThreadStarter.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/ThreadStarter.java deleted file mode 100644 index 590db63677..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/ThreadStarter.java +++ /dev/null @@ -1,229 +0,0 @@ -package north.gatherercrawler; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.util.*; -import north.gatherercrawler.util.CardsList; - -/** - * - * @author North - */ -public class ThreadStarter extends Thread { - - private static Integer threadsDone = 0; - private final Integer threads = 10; - private List sortedCards; - - public static synchronized void threadDone() { - threadsDone++; - } - - private void updateSortedCards() { - if (sortedCards == null) { - sortedCards = new ArrayList(); - Iterator iterator = CardsList.iterator(); - while (iterator.hasNext()) { - sortedCards.add(iterator.next()); - } - - Collections.sort(sortedCards, new Comparator() { - - public int compare(Card o1, Card o2) { - int expansionCompare = o1.getExpansion().compareTo(o2.getExpansion()); - return expansionCompare != 0 ? expansionCompare : o1.getCardNumber().compareTo(o2.getCardNumber()); - } - }); - } - } - - private void writeCardsToFile() { - try { - FileWriter fstream = new FileWriter("cards-data.txt"); - BufferedWriter out = new BufferedWriter(fstream); - Iterator iterator = sortedCards.iterator(); - while (iterator.hasNext()) { - out.write(iterator.next().toString()); - out.newLine(); - } - out.close(); - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - } - } - - private void writeCardsToUtilFile() { - try { - FileWriter fstream = new FileWriter("mtg-cards-data.txt"); - BufferedWriter out = new BufferedWriter(fstream); - Iterator iterator = sortedCards.iterator(); - while (iterator.hasNext()) { - Card card = iterator.next(); - String cardLine = generateUtilLine(card); - out.write(replaceSpecialChards(cardLine)); - out.newLine(); - - if (card.getOtherSide() != null) { - cardLine = generateUtilLine(card.getOtherSide()); - out.write(replaceSpecialChards(cardLine)); - out.newLine(); - } - } - out.close(); - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - } - } - - private void writeCardsToTrackerFile() { - Set knownSets = new TreeSet(); - knownSets.add("Fifth Edition"); - knownSets.add("Eighth Edition"); - knownSets.add("Ninth Edition"); - knownSets.add("Tenth Edition"); - knownSets.add("Magic 2010"); - knownSets.add("Magic 2011"); - knownSets.add("Magic 2012"); - knownSets.add("Planechase"); - knownSets.add("Duel Decks: Elspeth vs. Tezzeret"); - knownSets.add("Tempest"); - knownSets.add("Urza's Saga"); - knownSets.add("Urza's Legacy"); - knownSets.add("Invasion"); - knownSets.add("Planeshift"); - knownSets.add("Apocalypse"); - knownSets.add("Onslaught"); - knownSets.add("Mirrodin"); - knownSets.add("Darksteel"); - knownSets.add("Fifth Dawn"); - knownSets.add("Champions of Kamigawa"); - knownSets.add("Betrayers of Kamigawa"); - knownSets.add("Saviors of Kamigawa"); - knownSets.add("Ravnica: City of Guilds"); - knownSets.add("Guildpact"); - knownSets.add("Dissension"); - knownSets.add("Time Spiral"); - knownSets.add("Time Spiral \"Timeshifted\""); - knownSets.add("Planar Chaos"); - knownSets.add("Future Sight"); - knownSets.add("Lorwyn"); - knownSets.add("Morningtide"); - knownSets.add("Shadowmoor"); - knownSets.add("Eventide"); - knownSets.add("Shards of Alara"); - knownSets.add("Conflux"); - knownSets.add("Alara Reborn"); - knownSets.add("Zendikar"); - knownSets.add("Worldwake"); - knownSets.add("Rise of the Eldrazi"); - knownSets.add("Scars of Mirrodin"); - knownSets.add("Mirrodin Besieged"); - knownSets.add("New Phyrexia"); - knownSets.add("Innistrad"); - knownSets.add("Dark Ascension"); - knownSets.add("Avacyn Restored"); - try { - FileWriter fstream = new FileWriter("mtg-cards-tracker-data.txt"); - BufferedWriter out = new BufferedWriter(fstream); - Iterator iterator = sortedCards.iterator(); - while (iterator.hasNext()) { - Card card = iterator.next(); - if (knownSets.contains(card.getExpansion())) { - String cardLine = card.toString(); - out.write(replaceSpecialChards(cardLine)); - out.newLine(); - - if (card.getOtherSide() != null) { - cardLine = card.getOtherSide().toString(); - out.write(replaceSpecialChards(cardLine)); - out.newLine(); - } - } - } - out.close(); - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - } - } - - private String replaceSpecialChards(String original) { - return original.replace("\u2014", "-").replace("\u00E9", "e") - .replace("\u00C6", "AE").replace("\u00E6", "ae") - .replace("\u00C1", "A").replace("\u00E1", "a") - .replace("\u00C2", "A").replace("\u00E2", "a") - .replace("\u00D6", "O").replace("\u00F6", "o") - .replace("\u00DB", "U").replace("\u00FB", "u") - .replace("\u00DC", "U").replace("\u00FC", "u"); - } - - private String generateUtilLine(Card card) { - StringBuilder sb = new StringBuilder(); - sb.append(card.getName()).append("|"); - sb.append(card.getExpansion()).append("|"); - sb.append(card.getCardNumber() != null ? card.getCardNumber() : "").append("|"); - String rarity = card.getRarity() != null ? card.getRarity() : ""; - if (rarity.equalsIgnoreCase("Mythic Rare")) { - rarity = "M"; - } - if (rarity.equalsIgnoreCase("Rare")) { - rarity = "R"; - } - if (rarity.equalsIgnoreCase("Uncommon")) { - rarity = "U"; - } - if (rarity.equalsIgnoreCase("Common")) { - rarity = "C"; - } - if (rarity.equalsIgnoreCase("Basic Land")) { - rarity = "L"; - } - sb.append(rarity).append("|"); - List manaCost = card.getManaCost(); - for (String cost : manaCost) { - if (!cost.isEmpty()) { - sb.append("{").append(cost).append("}"); - } - } - sb.append("|"); - sb.append(card.getTypes()).append("|"); - String pts = card.getPowerToughness(); - if (pts != null && pts.length() > 1) { - String[] pt = pts.split("/"); - sb.append(pt[0].trim()).append("|"); - sb.append(pt[1].trim()).append("|"); - } else { - sb.append("||"); - } - List cardText = card.getCardText(); - for (int i = 0; i < cardText.size(); i++) { - sb.append(cardText.get(i)); - if (i < cardText.size() - 1) { - sb.append("$"); - } - } - sb.append("|"); - return sb.toString(); - } - - @Override - public void run() { - for (int i = 0; i < threads; i++) { - Thread t = new CardParser(); - t.start(); - } - - while (threads != threadsDone) { - try { - synchronized (this) { - this.wait(5000); - } - } catch (InterruptedException ex) { - } - } - - updateSortedCards(); - writeCardsToFile(); - writeCardsToUtilFile(); - writeCardsToTrackerFile(); - } -} diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/CardsList.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/CardsList.java deleted file mode 100644 index 7e6ea0086a..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/CardsList.java +++ /dev/null @@ -1,27 +0,0 @@ -package north.gatherercrawler.util; - -import java.util.Iterator; -import java.util.concurrent.ConcurrentSkipListSet; -import north.gatherercrawler.Card; - -/** - * - * @author North - */ -public class CardsList { - - private static final CardsList instance = new CardsList(); - private ConcurrentSkipListSet list; - - public CardsList() { - list = new ConcurrentSkipListSet(); - } - - public static void add(Card element) { - instance.list.add(element); - } - - public static Iterator iterator() { - return instance.list.iterator(); - } -} diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParseQueue.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParseQueue.java deleted file mode 100644 index c45b763095..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParseQueue.java +++ /dev/null @@ -1,29 +0,0 @@ -package north.gatherercrawler.util; - -import java.util.concurrent.ConcurrentLinkedQueue; - -/** - * - * @author North - */ -public class ParseQueue { - - private static final ParseQueue instance = new ParseQueue(); - private ConcurrentLinkedQueue queue; - - public ParseQueue() { - queue = new ConcurrentLinkedQueue(); - } - - public static void add(Integer element) { - instance.queue.add(element); - } - - public static Integer remove() { - return instance.queue.remove(); - } - - public static boolean isEmpty(){ - return instance.queue.isEmpty(); - } -} diff --git a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParsedList.java b/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParsedList.java deleted file mode 100644 index caa39038d7..0000000000 --- a/Utils/GathererCrawler/src/main/java/north/gatherercrawler/util/ParsedList.java +++ /dev/null @@ -1,25 +0,0 @@ -package north.gatherercrawler.util; - -import java.util.concurrent.ConcurrentSkipListSet; - -/** - * - * @author North - */ -public class ParsedList { - - private static final ParsedList instance = new ParsedList(); - private ConcurrentSkipListSet list; - - public ParsedList() { - list = new ConcurrentSkipListSet(); - } - - public static void add(Integer element) { - instance.list.add(element); - } - - public static boolean contains(Integer value) { - return instance.list.contains(value); - } -}