Index: /applications/editors/josm/plugins/wikipedia/images/dialogs/wikidata.svg =================================================================== --- /applications/editors/josm/plugins/wikipedia/images/dialogs/wikidata.svg (revision 31852) +++ /applications/editors/josm/plugins/wikipedia/images/dialogs/wikidata.svg (revision 31852) @@ -0,0 +1,6 @@ + + + + + + Index: /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/FetchWikidataAction.java =================================================================== --- /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/FetchWikidataAction.java (revision 31852) +++ /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/FetchWikidataAction.java (revision 31852) @@ -0,0 +1,147 @@ +// License: GPL. See LICENSE file for details./* +package org.wikipedia; + +import static org.openstreetmap.josm.tools.I18n.tr; +import static org.openstreetmap.josm.tools.I18n.trn; + +import java.awt.event.ActionEvent; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.openstreetmap.josm.Main; +import org.openstreetmap.josm.actions.JosmAction; +import org.openstreetmap.josm.command.ChangePropertyCommand; +import org.openstreetmap.josm.command.Command; +import org.openstreetmap.josm.command.SequenceCommand; +import org.openstreetmap.josm.data.osm.OsmPrimitive; +import org.openstreetmap.josm.gui.PleaseWaitRunnable; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.tools.MultiMap; + +public class FetchWikidataAction extends JosmAction { + + public FetchWikidataAction() { + super(tr("Fetch Wikidata IDs"), "dialogs/wikidata", + tr("Fetch Wikidata IDs using the ''wikipedia'' tag"), null, true); + } + + @Override + public void actionPerformed(ActionEvent e) { + if (getCurrentDataSet() == null) { + return; + } + Main.worker.submit(new Fetcher(getCurrentDataSet().getSelected())); + } + + static class Fetcher extends PleaseWaitRunnable { + private final Collection selection; + private boolean canceled = false; + private List commands; + + public Fetcher(Collection selection) { + super(tr("Fetching Wikidata IDs")); + this.selection = selection; + } + + @Override + protected void cancel() { + canceled = true; + } + + @Override + protected void realRun() { + final Map wikipediaByLanguage = getLanguageToArticlesMap(selection); + commands = new ArrayList<>(wikipediaByLanguage.keySet().size()); + getProgressMonitor().setTicksCount(wikipediaByLanguage.keySet().size()); + for (final Map.Entry i : wikipediaByLanguage.entrySet()) { + if (canceled) { + break; + } + final Command command = i.getValue().updateWikidataIds(getProgressMonitor().createSubTaskMonitor(1, false)); + if (command != null) { + commands.add(command); + } + } + } + + protected static Map getLanguageToArticlesMap(final Iterable selection) { + final Map r = new HashMap<>(); + for (final OsmPrimitive i : selection) { + final WikipediaApp.WikipediaLangArticle tag = WikipediaApp.WikipediaLangArticle.parseTag("wikipedia", i.get("wikipedia")); + if (tag != null) { + if (!r.containsKey(tag.lang)) { + r.put(tag.lang, new PrimitivesWithWikipedia(tag.lang)); + } + r.get(tag.lang).put(i, tag.article); + } + } + return r; + } + + @Override + protected void finish() { + if (!canceled && !commands.isEmpty()) { + Main.main.undoRedo.add(commands.size() == 1 ? commands.get(0) : new SequenceCommand(tr("Add Wikidata"), commands)); + } + } + } + + static class PrimitivesWithWikipedia { + final String lang; + final MultiMap byArticle = new MultiMap<>(); + + public PrimitivesWithWikipedia(String lang) { + this.lang = lang; + } + + public void put(OsmPrimitive key, String wikipedia) { + byArticle.put(wikipedia, key); + } + + protected Command updateWikidataIds(ProgressMonitor monitor) { + final int size = byArticle.keySet().size(); + monitor.beginTask(trn( + "Fetching {0} Wikidata ID for language ''{1}''", + "Fetching {0} Wikidata IDs for language ''{1}''", size, size, lang)); + final List commands = new ArrayList<>(size); + final Map wikidataByWikipedia = WikipediaApp.getWikidataForArticles(lang, byArticle.keySet()); + for (Map.Entry> i : byArticle.entrySet()) { + final String wikipedia = i.getKey(); + final String wikidata = wikidataByWikipedia.get(wikipedia); + if (wikidata != null) { + commands.add(new ChangePropertyCommand(i.getValue(), "wikidata", wikidata)); + } else { + Main.warn(tr("No Wikidata ID found for {0}", lang + ":" + wikipedia)); + } + } + monitor.finishTask(); + return commands.isEmpty() + ? null + : new SequenceCommand(tr("Add Wikidata for language ''{0}''", lang), commands); + } + } + + @Override + protected void updateEnabledState() { + if (getCurrentDataSet() == null) { + setEnabled(false); + } else { + updateEnabledState(getCurrentDataSet().getSelected()); + } + } + + @Override + protected void updateEnabledState(Collection selection) { + for (final OsmPrimitive i : selection) { + if (i.hasKey("wikipedia")) { + setEnabled(true); + return; + } + } + setEnabled(false); + } +} Index: /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java =================================================================== --- /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java (revision 31851) +++ /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaApp.java (revision 31852) @@ -16,4 +16,5 @@ import java.util.Objects; import java.util.Scanner; +import java.util.TreeMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -32,4 +33,5 @@ import org.openstreetmap.josm.tools.Utils.Function; import org.w3c.dom.Document; +import org.w3c.dom.Node; import org.w3c.dom.NodeList; @@ -164,4 +166,39 @@ } + /** + * Returns a map mapping wikipedia articles to wikidata ids. + */ + static Map getWikidataForArticles(String wikipediaLang, Collection articles) { + try { + final String url = "https://www.wikidata.org/w/api.php" + + "?action=wbgetentities" + + "&props=sitelinks" + + "&sites=" + wikipediaLang + "wiki" + + "&sitefilter=" + wikipediaLang + "wiki" + + "&format=xml" + + "&titles=" + Utils.join("|", Utils.transform(articles, new Function() { + @Override + public String apply(String x) { + return Utils.encodeUrl(x); + } + })); + Main.info("Wikipedia: GET " + url); + final Map r = new TreeMap<>(); + try (final InputStream in = Utils.openURL(new URL(url))) { + final Document xml = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(in); + final NodeList nodes = (NodeList) XPathFactory.newInstance().newXPath().compile("//entity").evaluate(xml, XPathConstants.NODESET); + for (int i = 0; i < nodes.getLength(); i++) { + final Node node = nodes.item(i); + final String wikidata = (String) XPathFactory.newInstance().newXPath().compile("./@id").evaluate(node, XPathConstants.STRING); + final String wikipedia = (String) XPathFactory.newInstance().newXPath().compile("./sitelinks/sitelink/@title").evaluate(node, XPathConstants.STRING); + r.put(wikipedia, wikidata); + } + } + return r; + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + static Collection getInterwikiArticles(String wikipediaLang, String article) { try { Index: /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaPlugin.java =================================================================== --- /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaPlugin.java (revision 31851) +++ /applications/editors/josm/plugins/wikipedia/src/org/wikipedia/WikipediaPlugin.java (revision 31852) @@ -14,4 +14,5 @@ new WikipediaCopyTemplate(); MainMenu.add(Main.main.menu.dataMenu, new WikipediaAddNamesAction()); + MainMenu.add(Main.main.menu.dataMenu, new FetchWikidataAction()); } Index: /applications/editors/josm/plugins/wikipedia/test/unit/org/wikipedia/WikipediaAppTest.java =================================================================== --- /applications/editors/josm/plugins/wikipedia/test/unit/org/wikipedia/WikipediaAppTest.java (revision 31851) +++ /applications/editors/josm/plugins/wikipedia/test/unit/org/wikipedia/WikipediaAppTest.java (revision 31852) @@ -13,4 +13,5 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import static org.hamcrest.CoreMatchers.hasItem; @@ -131,3 +132,14 @@ })); } + + @Test + public void testGetWikidataForArticles() throws Exception { + final Map map = WikipediaApp.getWikidataForArticles("en", + Arrays.asList("London", "Vienna", "Völs, Tyrol", "a-non-existing-article")); + assertThat(map.get("London"), is("Q84")); + assertThat(map.get("Vienna"), is("Q1741")); + assertThat(map.get("Völs, Tyrol"), is("Q278250")); + assertThat(map.get("a-non-existing-article"), nullValue()); + assertThat(map.size(), is(4)); + } }