Final Release 1.0

dstN · Apr 8, 2020 · 6cba128 · 6cba128
1 parent 1ff8057
commit 6cba128
Showing 3 changed files with 54 additions and 115 deletions.
diff --git a/README.md b/README.md
@@ -1,15 +1,12 @@
 # tmScrape
 local web scraping of the (max first 25 entries) transfer records from transfermarkt.com
 # usage
-just clone/download the repository and open index.html in your browser.
+Just clone/download the repository and open index.html in your browser.
 
-Wait until your browser doesn't load anymore.
+Wait until "Scarping done!" appears and click on "Copy to clipboard"
 
-Add this two line of code to your console-drawer:
+It takes a moment until it copies everything. If it's done copying you get an alert.
 
-``` $(".before>p").not(":last-of-type").append("<span>,</span"); ```
-
-``` $(".after:last-of-type").empty().append("<span>]</span>"); ```
 # format
 every player comes in a json-object. with following data within:
 * Rank within his Transfer-Year

diff --git a/index.html b/index.html
@@ -8,9 +8,16 @@
         visibility: hidden !important;
       }
       .table {
+        display: none !important;
+        visibility: hidden !important;
+      }
+      .copy {
         display: block !important;
         visibility: visible !important;
       }
+      .copy #copy, .copy button {
+        display: none;
+      }
     </style>
     <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.4.1/jquery.min.js"></script>
   </head>
@@ -19,24 +26,23 @@
 
     </div>
     <div class="table">
-      <table>
-        <thead>
-
-        </thead>
-        <tbody>
-
-        </tbody>
-      </table>
-      <div class="start">
+      <span class="start">
         {
-      </div>
+      </span>
       <div class="json">
 
 
       </div>
-      <div class="end">
+      <span class="end">
         }
-      </div>
+      </span>
+    </div>
+    <div class="copy">
+      <textarea id="copy" rows="12" cols="100"></textarea>
+      <br>
+      <button onclick="copyFunction()">
+        Copy to clipboard
+      </button>
     </div>
     <script type="text/javascript" src="main.js"></script>
   </body>

diff --git a/main.js b/main.js
@@ -1,24 +1,22 @@
 var startDate = 1956; // first year a transfer fee was recorded on transfermarkt.com
 var endDate = new Date().getFullYear(); // current Year
 var getUrl = "https://www.transfermarkt.com/transfers/transferrekorde/statistik/top/saison_id/"; // ADD Year after saison_id/
-var yearIterator = startDate;
-function ajaxCall(){
+function ajaxCall(iterate){
   $.ajax({
-    url: getUrl + yearIterator,
+    url: getUrl + iterate,
     type: 'GET',
     beforeSend: function() {
-      console.log("Loading: "+yearIterator);
+      $(".copy").prepend('<p class="loadingScrape">Scraping...</p>');
     },
     complete: function() {
 
     },
     success: function(data) {
-      var currentYearString = '"'+yearIterator+'"';
-      $(".json").append('<div class="'+yearIterator+' before">'+currentYearString+': [ </div>');
-      $(".json").append('<div class="'+yearIterator+' after"></div>');
-      $(".loader").html(data); // load i-year into .loader-Element
-      var tableRows = $(".loader").find(".items>tbody>tr"); // select every table-row (player)
-      tableRows.each(function(){ // function for every player within table
+      var currentYearString = '"'+iterate+'"';
+      $(".json").append('<div class="'+iterate+' before">'+currentYearString+': [ </div>');
+      $(".json").append('<div class="'+iterate+' after"></div>');
+      var tableRows = $(data).find(".items>tbody>tr");
+      tableRows.each(function() {
         var rank = $(this).find(">td:first-of-type").text(); // get rank within year
         var picture = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:first-of-type>td:first-of-type>img").attr("src"); // get playerpicture
         var playerLink = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:first-of-type>td.hauptlink>a").attr("href"); //get playerLink to transfermarkt.com
@@ -84,97 +82,35 @@ function ajaxCall(){
           transferHistoryLink: "https://www.transfermarkt.com" + transferHistoryLink
         };
         var playerJSON = JSON.stringify(playerObj); // object to JSON
-        $("."+year+".before").append("<p id='"+playerID+"'>"+playerJSON+"</p>"); // parse json to browser
+        $("."+iterate+".before").append("<p id='"+playerID+"'>"+playerJSON+"</p>"); // parse json to browser
       });
-      $(".loader").empty();
-      $("."+yearIterator+".after").append("<span>],</span>");
+      $("."+iterate+".after").append("<span>],</span>");
 
-      if(yearIterator++ < 1960) { ajaxCall(); console.log("I called again"); }
-      else{ console.log("I elsed again"); }
-    }
-  });
-}
-ajaxCall();
-/* for (i=startDate; i <= endDate; i++) { // iterate until current year - change to endDate in production mode
-  var currentYear = i;
-  var currentYearString = '"'+i+'"';
-  console.log('Line ' + i);
-  $(".json").append('<div class="'+currentYear+' before">'+currentYearString+': [ </div>');
-  $(".json").append('<div class="'+currentYear+' after"></div>');
-  $.get(getUrl + currentYear, function(data){}).done(function(data) { // get the page of current year
-    console.log('Get-Line ' + currentYear);
-    $(".loader").html(data); // load i-year into .loader-Element
-    var tableRows = $(".loader").find(".items>tbody>tr"); // select every table-row (player)
-    tableRows.each(function(){ // function for every player within table
-      var rank = $(this).find(">td:first-of-type").text(); // get rank within year
-      var picture = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:first-of-type>td:first-of-type>img").attr("src"); // get playerpicture
-      var playerLink = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:first-of-type>td.hauptlink>a").attr("href"); //get playerLink to transfermarkt.com
-      var playerID = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:first-of-type>td.hauptlink>a").attr("id"); // get playerID
-      var playerName = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:first-of-type>td.hauptlink>a").text(); // get playerName
-      var position = $(this).find(">td:nth-of-type(2)").find("table>tbody>tr:last-of-type>td").text(); // get players position
-      var year = $("select[name='saison_id'] option[selected='selected']").val(); // get year of transfer
-      var yearLink = getUrl + year; // get yearLink of transfer
-      var nationalities = $(this).find(">td:nth-of-type(4)>img"); // get nationalities
-      if(nationalities.length===2){ // check if there are 2 nationaities
-        var nationalityOne = nationalities.eq(0).attr("alt"); // get both nationalities
-        var nationalityTwo = nationalities.eq(1).attr("alt");
-        var flagOne = nationalities.eq(0).attr("src");
-        var flagTwo = nationalities.eq(1).attr("src");
+      if(iterate++ < endDate) {
+        ajaxCall(iterate);
       }
       else {
-        var nationalityOne = nationalities.attr("alt"); // get just first nationality
-        var flagOne = nationalities.attr("src");
-        var nationalityTwo = "none"; // default value for 2nd nationality
-        var flagTwo = "none";
+        $(".before>p").not(":last-of-type").append("<span>,</span");
+        $(".after:last-of-type").empty().append("<span>]</span>");
+        var completeJSON = $(".table").text();
+        var textarea = $("#copy");
+        textarea.val(completeJSON);
+        $(".loadingScrape").hide(400);
+        $(".copy").prepend("<p>Scraping done!</p>");
+        $(".copy button").show(400);
       }
-      var joinedClub = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:first-of-type>td:last-of-type>a").text(); // get joinedClub
-      var joinedClubLink = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:first-of-type>td:last-of-type>a").attr("href"); // get the transfermarkt.com link to the club
-      var clubEmblem = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:first-of-type>td:first-of-type>a>img").attr("src"); // get emblem of clb
-      var leagueNationality = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:last-of-type>td:first-of-type>img").attr("alt"); // get nationality of joined league
-      var leagueNationalityFlag = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:last-of-type>td:first-of-type>img").attr("src"); // get flag of joined leagues nationality
-      var leagueName = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:last-of-type>td:first-of-type>a").text(); // get leagueName of joined league
-      var leagueLink = $(this).find(">td:nth-of-type(5)").find("table>tbody>tr:last-of-type>td:first-of-type>a").attr("href"); // get leagueLink of joined league
-      var transferFee = $(this).find(">td:last-of-type>a").text(); // get transfer fee
-      var feeFull = transferFee.replace("€", "");
-      if(transferFee.indexOf('k') > -1) {
-        feeFull = feeFull.replace('k', '');
-        feeFull = feeFull * 1000
-      }
-      else if(transferFee.indexOf('m') > -1) {
-        feeFull = feeFull.replace('m', '');
-        feeFull = feeFull * 1000000
-      };
-      var transferHistoryLink = $(this).find(">td:last-of-type>a").attr("href"); // get transfer history link
-
-      var playerObj =
-      {
-        rank: rank,
-        picture: picture,
-        playerLink: "https://www.transfermarkt.com" + playerLink,
-        playerName: playerName,
-        position: position,
-        year: year,
-        yearLink: yearLink,
-        nationalityOne: nationalityOne,
-        nationalityTwo: nationalityTwo,
-        flagOne: flagOne,
-        flagTwo: flagTwo,
-        joinedClub: joinedClub,
-        joinedClubLink: "https://www.transfermarkt.com" + joinedClubLink,
-        clubEmblem: clubEmblem,
-        leagueNationality: leagueNationality,
-        leagueNationalityFlag: leagueNationalityFlag,
-        leagueName: leagueName,
-        leagueLink: "https://www.transfermarkt.com" + leagueLink,
-        transferFee: feeFull,
-        transferFeeToolTip: transferFee,
-        transferHistoryLink: "https://www.transfermarkt.com" + transferHistoryLink
-      };
-      var playerJSON = JSON.stringify(playerObj); // object to JSON
-      $("."+year+".before").append("<p id='"+playerID+"'>"+playerJSON+"</p>"); // parse json to browser
-    });
-    $(".loader").empty();
+    },
+    error: function(error) {
+      console.log("Error: "+error);
+    }
   });
-  $("."+currentYear+".after").append("<span>],</span>");
 }
-*/
+ajaxCall(startDate);
+
+function copyFunction() {
+  $(".copy #copy").show(400);
+  var copy = $("#copy");
+  copy.select();
+  document.execCommand("copy");
+  alert("Copied to clipboard!");
+}