User:Gryllida/js/addSourcesUi-checks.js

/*
Author : Svetlana Tkachenko svetlana@members.fsf.org
This file is a part of addSourcesUi.
Licence: GPLv3+
Description: checks citoid output against source template information provided by wiki
TODO: [ ] share some subroutines with User:Gryllida/js/addSourcesUi.js
*/ 

var monthNames = [
    "January", "February", "March",
    "April", "May", "June", "July",
    "August", "September", "October",
    "November", "December"
  ];
var api = new mw.Api();



var getLocation = function(href) {
	var l = document.createElement("a");
	l.href = href;
	return l;
};
var getDomain = function (url){
	var l = getLocation(url);
	return l.hostname;
}
var errors = {};
var articlesChecked=0;

var templatesChecked=0;
var templatesCheckedWithError=0;
var templatesTotal=0;


function _getCitoidApiUrl(url){
	var apiUrl = mw.config.get ('wgServer') ;
	    apiUrl = apiUrl + '/api/rest_v1/data/citation/mediawiki/';
	    apiUrl = apiUrl + encodeURIComponent(url);
	return apiUrl;
}

function handleError(thissourceinfo, thissourceinfo2, data, sourceurl, pageName){
	var domain = getDomain(sourceurl);
	console.log(domain + ' - ' + pageName);
	var citoiduri = _getCitoidApiUrl(sourceurl);
	
	// add the errors to output ('errors' var)
	if(!errors[domain]){
		errors[domain]={
			'title mismatch': {},
			'author mismatch': {},
			'publisher mismatch': {},
			'date mismatch': {}
		};
	}
	// identify relevant errors
	if(thissourceinfo.title != thissourceinfo2.title){
		errors[domain]['title mismatch'][sourceurl] = {
			'wiki': thissourceinfo.title,
			'citoid': thissourceinfo2.title
		};
	}
	if(thissourceinfo.author != thissourceinfo2.author){
		errors[domain]['author mismatch'][sourceurl] = {
			'wiki': thissourceinfo.author,
			'citoid': thissourceinfo2.author
		};
	}
	if(thissourceinfo.pub != thissourceinfo2.pub){
		errors[domain]['publisher mismatch'][sourceurl] = {
			'wiki': thissourceinfo.pub,
			'citoid': thissourceinfo2.pub
		};
	}
	if(thissourceinfo.date != thissourceinfo2.date){
		errors[domain]['date mismatch'][sourceurl] = {
			'wiki': thissourceinfo.date,
			'citoid': thissourceinfo2.date
		};
	}
}
  
function _getGetSourcesFromArticle(pageName){
	//console.log('getting sources from '+pageName);
	//https://en.wikipedia.org/w/api.php?action=parse&page=New_York&format=json&prop=wikitext&section=4
	//http://en.wikipedia.org/w/api.php?format=json&action=parse&prop=sections&page=New_York
	api.get({
		action: 'parse',
		prop: 'sections',
		page: pageName
	}).done( function ( data ) {
		$(data.parse.sections).each(function(i){
			if(this.line=='Sources'){
				api.get({
					action: 'parse',
					page: pageName,
					prop: 'parsetree',
					section: this.index
				}).done( function ( data ) {
					var xml = data.parse.parsetree['*'],
						xmlDoc = $.parseXML( xml ),
						$xml = $( xmlDoc ),
						$templates = $xml.find( "template" );
					var sourceinfo = [];
					$templates.each(function(i){
						var templateTitle = $.trim($(this).find('title').text());
						//alert(':'+templateTitle+":");
						if(templateTitle=='source'){
							templatesTotal += 1;
							var thissourceinfo = {};
							var parts = $(this).find('part');
							parts.each(function(i){
								var partName = $.trim($(this).find('name').text());
								var partValue = $.trim($(this).find('value').text());
								thissourceinfo[partName] = partValue;
							})
							
							var sourceurl = thissourceinfo['url'];
							//console.log(sourceurl);
							_getPublicationData1(sourceurl).then(function(data) {
								thissourceinfo2 = _getPublicationData2(data, sourceurl);
								templatesChecked += 1;
								if(thissourceinfo != thissourceinfo2){
									handleError(thissourceinfo, thissourceinfo2, data, sourceurl, pageName);
								}
								if(templatesChecked+templatesCheckedWithError==templatesTotal){
									console.log('finished all articles');
									delete errors['query']; 
									console.log(errors);
									api.postWithToken("edit", {
										action: 'edit',
										title: 'User:Gryllida/js/addSourcesUi.json',
										text: JSON.stringify(errors),
										summary: 'update errors ([[User:Gryllida/js/addSourcesUi.js|assisted]])'
									}).done(function (data){
										console.log('saved errors I think - everything complete');
									}).fail( function ( code, result ) {
										alert('3'); // is not reached
										if ( result.errors && result.errors[0] ) {
											errorMessage = result.errors[0].html;
										} else {
											errorMessage = 'API hiba: ' +  code;
										}
										alert( errorMessage );
									} );
								}
							}).catch(err => {
								templatesCheckedWithError += 1;
								//console.log('error', err)
							});
							
							sourceinfo.push(thissourceinfo);
							//console.log(sourceinfo);
						}
					});
					console.log(errors);
					articlesChecked +=1;
				});
			}
		})
	});
}


function _getPublicationData1(url){
	//console.log('getting the url '+ url);
	var apiUrl = _getCitoidApiUrl(url);
	//console.log(apiUrl);
	return $.get(apiUrl);
}

function _getPublicationData2(data, url){
	// Log
	//console.log('got the data: ');
	//console.log(url);
	//console.log(data);
	
	// Get author
	var authors = data[0].author;
	var authorStrings = [];
	$(authors).each(function( index ) {
		authorStrings.push(this[0] + ' ' + this [1]);
	});
	var author = authorStrings.join (', ');
	//console.log('author: '+ author);
	
	// Get publisher, title
	var publisher = data[0].publicationTitle || "";
	var title = data[0].title;
	
	// Get date (in the correct format)
	var datestring = data[0].date; // 2020-12-31
	var date="";
	if(datestring){
		var dateOrig = datestring.split('-'); // 2020-12-31
		var dateYear = dateOrig[0];
		var dateMonth = monthNames[dateOrig[1]-1];
		var dateDay = parseInt(dateOrig[2],10)
		date = dateMonth + ' ' + dateDay + ', ' + dateYear;
	}
	var sourceinfo = {
		url: url,
		author: author,
		pub: publisher,
		title: title,
		date: date
	};
	//console.log(sourceinfo);
	return sourceinfo;
	
}

var articlesToCheck = [
"Iran: Wreckage found of plane crashed in mountains; all believed dead",
"Oxfam produces action plan after revelation of prostitution scandal on Haiti",
"Fourth U.S. state governor orders net neutrality in government contracts",
"United States: Berkeley, California declares itself a sanctuary city for recreational cannabis",
"United States: Jet loses engine cover over Pacific en route to Honolulu from San Francisco",
"Russia: ex-governor of Sakhalin region Alexander Khoroshavin sentenced to thirteen years for bribery",
"Former Irish footballer Liam Miller dies at 36",
"Competition Commission of India fines Google ₹1.36 billion for 'search bias'",
"Pakistan court sentences one man to death penalty, and life imprisonment to five others for Mashal Khan lynching incident",
"Dhaka court sentences former Bangladeshi Prime Minister Khaleda Zia to five years on corruption charges",
"Poet, lyricist, and digital activist John Perry Barlow dies, aged 70",
"Chinese Foreign Ministry confirms arrest of bookseller Gui Minhai",
"Dutch Football Association appoints Ronald Koeman manager of men's national team",
"Margot Duhalde, Chile's first female military pilot, dies aged 97",
"SpaceX Falcon Heavy rocket blasts Elon Musk's personal Tesla into solar orbit",
"United States: Two killed, more than a hundred injured in Amtrak train collision in South Carolina",
"North Korea to send head of state to South Korea for Olympics",
"United States: Four injured in Los Angeles school shooting",
"India: Head of Delhi Commission on Women calls for prompt capital punishment for child rape",
"Football: Giroud leaves Arsenal to sign eighteen-month deal with cross-town rivals Chelsea",
"Football: Arsenal signs Aubameyang from Dortmund",
"Football: Manchester United announces extending Juan Mata's contract",
"Koreas hold joint training session for Olympics",
"Football: Barça centre-back Piqué extends contract until 2022",
"USA Gymnastics board resigning after sex abuse",
"Afghanistan: Ambulance suicide attack kills about 100 people in Kabul",
"United States: Dr. Larry Nassar sentenced in sexual abuse case",
"India: Karnataka closed for a day for protests over Mahadayi river share",
"South Korea: Fire in hospital housing elderly people kills at least 37",
"United States: Two dead in Kentucky high school shooting",
"'Save the Children' organisation suspends all activities in Afghanistan after ISIL attack",
"Healthy cloned monkeys born in Shanghai",
"US Vice President Pence speaks to Israeli Knesset: US embassy to move to Jerusalem",
"Chile: Pope Francis stirs outrage with 'slander' comment",
"Ukraine passes bill on war-torn eastern regions",
"Zimbabwean politician Bennett and four others die in New Mexico helicopter crash",
"Irish rock band The Cranberries' lead singer Dolores O'Riordan dies at 46",
"United States: One person dead after boat to offshore casino burns off Florida coast",
"British surfers catch more than waves: Scientists find antibiotic-resistant bacteria",
"Airborne sedan smashes into dental office in Santa Ana, California, US",
"United States: State of Hawaii criticized by head of Federal Communications Commission over incoming missile alert mistake",
"BBC newsreader Alagiah to undergo treatment for bowel cancer",
"Turkey: Aircraft skids off runway toward Black Sea",
"United States: At least fifteen dead in Southern California after rain causes mudslides and floods",
"Israel issues travel ban on 20 non-government organizations over pro-Palestinian boycotts",
"Saudi Arabia: Princes arrested after protesting austerity",
"Mysterious dimming of Tabby's star likely due to space dust, not alien superstructures, say scientists",
"England: Multi-storey carpark in Liverpool gutted by fire, 1,300 vehicles destroyed",
"Tennis: Andy Murray withdraws from Australian Open",
"Scaffolding collapses in Copenhagen",
"United States: New York City mayoral swear-in to pair potential 2020 presidential prospects",
"Football: Liverpool reaches agreement with Southampton FC to sign van Dijk",
"Israel transport minister Katz plans to name a railway station after US president Donald Trump",
"Belgium stops telegram services",
"Liberia: Former football striker George Weah wins presidential election",
"Afghanistan: More than 40 killed in explosions in Kabul",
"Israeli chess players barred at chess tournament in Saudi Arabia",
"Russia: Runaway bus kills at least four in entrance to Moscow Metro station",
"Guatemalan president Jimmy Morales announces move of Israeli embassy from Tel Aviv to Jerusalem",
"England: Fire at London Zoo kills aardvark, meerkats believed dead",
"England: Baby born with heart outside body operated on; surviving, three weeks after birth",
"Apple, Inc. confirms acquisition of Shazam",
"UEFA Champions League 2017–2018: Draw for Last 16 held at Nyon",
"Football: Peter Stöger replaces Peter Bosz as Borussia Dortmund's boss",
"Wikinews attends ComicCon in Bangalore, India",
"Yemen: Former president Ali Abdullah Saleh killed in Sana'a by Houthi militants",
"Football: Italian club AC Milan sacks Vincenzo Montella as manager",
"Israeli Health minister Ya'akov Litzman resigns in protest after Jews made to work on Jewish rest day",
"Football: Messi signs contract extension with FC Barcelona",
"Time magazine refutes US President Donald Trump's Twitter claim he was nominated Time 'Person of the Year'",
"Italian court sentences Brazilian footballer Robinho to nine years for 2013 sexual assault",
"Researchers report rapid formation of new bird species in Galápagos islands",
"Singapore announces driverless buses on public roads from 2022",
"Zimbabwe: Robert Mugabe resigns presidency after military coup, threat of impeachment",
"Charles Manson, serving nine life sentences for 1969 murders, dies aged 83",
"Gunman kills at least four in shooting in Rancho Tehama, California",
"DR Congo: Train crash kills more than 30 in Lualaba province",
"Poland: Thousands of far-right nationalists gather in Warsaw to march for white supremacy, anti-liberalism, and anti-Islam on Polish independence day",
"Qatar appoints four women to its law-drafting Shura Council",
"Turkey, US embassies resume issuing non-immigrant visas",
"Italian footballer Andrea Pirlo announces retirement",
"Kazakhstan: President Nazarbayev signs decree to change Kazakh characters from Cyrillic to Latin-based script",
"U.S. government report says climate change is human-made",
"Tennis: Nadal withdraws from Paris Masters, suffers from knee injury",
"Astronomers report dwarf star with unexpectedly giant planet",
"Puerto Rico power company cancels US'300 million Whitefish contract",
"CMHC: housing market in Canada 'highly vulnerable'",
"United States Senator Jeff Flake announces retirement, citing 'profoundly misguided' party politics",
"Five United States ex-presidents raise relief funds at hurricane event",
"United States judges block third version of President Trump's travel ban",
"Poland: 27-year-old arrested in Stalowa Wola for stabbing eight people",
"Pakistan: car rams into police truck killing at least seven, injuring 22 in Quetta",
"Arrangement of light receptors in the eye may cause dyslexia, scientists say",
"Digital security researchers publicly reveal vulnerability in WPA2 WiFi protocol",
"Iraqi army regains control over Peshmerga fighters' occupied Kirkuk",
"Israeli athletes barred from representing country in Abu Dhabi's grand slam",
"Austrian People's Party wins majority national election, Sebastian Kurz to become world's youngest national leader at 31",
"Abducted Canadian-US couple recovered from Pakistan's tribal areas",
"Greek parliament votes to legalise sex-change without operation",
"Football: Robben retires from international football after Netherlands fails to qualify for World Cup",
"As shipping exemption expires, hurricane-torn Puerto Rico may face changes in relief from mainland United States",
"Australia bars North Korea's U-19 football team from entering country citing 'illegal nuclear programmes'",
"Hurricane Nate weakens as it reaches United States",
"Football: Lewandowski's hat-trick against Armenia makes him Poland's top-scorer, close to World Cup qualification",
"Pakistan: At least 18 killed, others wounded in suicide attack at Balochistan Sufi shrine",
"Researchers find preserving spotted owl habitat may not require a tradeoff with wildfire risk after all",
"Canadian government settles lawsuit over children 'scooped' out of indigenous communities",
"Football: FC Bayern, Jupp Heynckes reach agreement; club's head coach until season end",
"In Malaysia's high court, pathologist testifies Kim Jong Nam was killed by weapon of mass destruction",
"US rock artist Tom Petty dies at 66",
"Suspects in slaying of pregnant North Dakota woman enter not guilty pleas",
"Russia asks Facebook to comply with personal data policy",
"Football: Asensio agrees to sign contract extension with Real Madrid",
"Football: Varane agrees to sign contract extension with Real Madrid",
"Saudi Arabia to allow women to drive",
"British actor Tony Booth dies at 85",
"First Star Trek series in twelve years, Discovery, debuts on television",
"Uber London to lose operator licence after September",
"Football: Benzema signs contract extension with Real Madrid",
"US toy retail giant Toys R Us files for bankruptcy in US, Canada",
"Al Jazeera says Snapchat's act a 'clear attack on the rights of journalists' as Snapchat blocks Qatar-based news network's channel in Saudi Arabia",
"Football: German goalkeeper Manuel Neuer sidelined until 2018 after leg injury",
"Football: Carvajal signs contract extension with Real Madrid",
"Football: Isco signs contract extension with Real Madrid",
"Football: Roy Hodgson pens two-year contract with Crystal Palace; becomes seventh permanent manager of Palace in six years",
"Device explodes on London Tube train at Parsons Green",
"Football: Marcelo signs contract extension with Real Madrid",
"Football: Crystal Palace sacks de Boer after 77 days as manager",
"After killing wife and children, police officer commits suicide in Noyon, France",
"Fox News cuts ties with journalist Eric Bolling amid sexual misconduct claims; his son dies hours later",
"Fifteen states sue United States President Donald Trump for cancelling program for undocumented immigrant minors",
"Peggy Whitson, record-breaking 'American space ninja', returns to Earth",
"Tim Curry, TV premiere screenings, cosplay feature at Fan Expo Canada",
"Frankfurt defuses World War II-era bomb, evacuates 60,000",
"Chile president Michelle Bachelet signs bill to legalise same-sex marriage",
"Many in Texas still reeling following hurricane Harvey",
"California Supreme Court upholds law hastening death penalty",
"England's top scorer Wayne Rooney retires from international football",
"Man with knife stabs people in Finnish city of Turku",
"New South Wales police extradict 'self-healer' Hongchi Xiao from London over death of six-year-old boy at conference",
"Ohio man charged with second-degree murder of Charlottesville, Virginia counter-protester",
"Mozilla, Creative Commons, Wikimedia Foundation announce Bassel Khartabil Free Culture fellowship following execution of open culture activist",
"Australia: Victorian government to trial driverless vehicles on public roads",
"Former Governor of Texas Mark White dies aged 77",
"Bayern Munich's technical director Michael Reschke leaves the club, becomes sporting director of VfB Stuttgart",
"Football: Bayern beats Borussia Dortmund 5-4 on penalties to win German Super Cup",
"Prince Philip of UK makes last solo public engagement after 65 years",
"Football: Paris Saint-Germain pays world record €222m to sign Neymar",
"Anthony Scaramucci leaves role as US White House communications director after ten days",
"Wikinews interviews producer of horror film '6:66PM'",
"British dancer and talent show winner Robert Anker dies in car accident aged 27",
"Publisher withdraws book about Nelson Mandela's final days after family complaint",
"Polish President Andrzej Duda vetoes law placing Supreme Court under power of ruling party",
"Linkin Park's lead singer Chester Bennington dies at 41",
"Football: Spanish striker Morata joins Chelsea",
"After visa snags, all-girl Afghan team honored for 'courageous achievement' at international robotics competition",
"Visa now compulsory for Qataris to enter Egypt, foreign ministry says",
"Sun's mood swings not so strange after all, say scientists",
"Nobel Peace Prize winning Chinese dissident Liu Xiaobo dies, aged 61",
"Costa joins Juventus FC on one-year loan",
"Djokovic withdraws from Wimbledon Championships",
"Astronomers discover smallest known star",
"Football's Rooney returns to Everton",
"India Supreme Court overrules High Court: rivers Yamuna, Ganga no longer living entities",
"Volvo announces all new car models electric or hybrid from 2019",
"Beachside photos of New Jersey governor draw criticism, photo-mashups during state budget crisis",
"Terry joins Aston Villa",
"German Bundestag votes for same-sex marriage",
"Third Trump travel ban takes effect",
"Czech parliament votes in favour of legalising firearms possession",
"Thousands gather in Jantar Mantar and other cities to protest against mob violence",
"Police stop LGBT march in Istanbul for third consecutive year",
"Curaçao wins maiden Caribbean Cup",
"Ohio judge declares mistrial for officer who shot Samuel DuBose",
"Karen Handel wins runoff for seat in United States House of Representatives",
"Bihar class X student allegedly gang raped, thrown from moving train",
"Judge declares mistrial in Bill Cosby sexual assault case",
"South Korea and Stielike part ways",
"Academy Award-winning director John G. Avildsen dies aged 81",
"Amazon.com to acquire Whole Foods at US'42 per share",
"Leo Varadkar becomes first openly gay elected Taoiseach of Ireland, succeeds Enda Kenny",
"Shooter targets Congressional baseball practice in Virginia, six hospitalized",
"Bayern Munich signs Corentin Tolisso from Lyon",
"¡La décima! Nadal again a French CHAMP10N",
"Batman star Adam West dies aged 88",
"Japan's National Diet passes law allowing Emperor Akihito to abdicate within three years",
"Theresa May's Conservative Party wins UK election but loses majority, leaving Brexit plan in question",
"Former U.S. FBI Director James Comey testifies about President Trump",
"Borussia Dortmund announces former Ajax coach Peter Bosz as team's new boss",
"Astronomers reveal discovery of the hottest gas giant exoplanet known yet",
"Real Madrid defends Champions League title",
"Conductor Jeffrey Tate dies aged 74",
"Seven killed, forty-eight injured in attack on London Bridge",
"Curiosity Rover analysis suggests chemically complex lake once graced Mars's Gale crater",
"Death toll exceeds 200 after heavy rain and mudslides in Sri Lanka",
"Man posthumously marries Legion of Honour recipient in France",
"For fans, by fans: Toronto anime event 2017 among continent's largest",
"Truck bomb kills at least 80 in Afghan capital city center",
"Portland man arrested for murder, intimidation in MAX train hate speech incident",
"Same-sex spouse of Luxembourg's Prime Minister poses with other spouses of world leaders in Brussels photo op"
];


function getStarted(){
	api.get(
			{
				titles: 'User:Gryllida/js/addSourcesUi.json',
				'action': 'query',
				'prop': 'revisions',
				'rvprop': 'content',
				'format': 'json',
				'origin': '*' // Allow requests from any origin so that this script can be used on localhost and non-Wikimedia sites
			}
		).done( function ( data ) {
			errors = data;
			$(articlesToCheck).each(
				function(i){
					_getGetSourcesFromArticle(this);
				}
			);
		});
}

var link = mw.util.addPortletLink(mw.config.get('skin') === 'vector' ? 'p-views' : 'p-cactions',
		'javascript:void(0);', 'Check sources', 'p-gryllida-checksources', 'Checks sources from wiki articles and from citoid', '6'
);

link.addEventListener('click', getStarted);