Module:DescriptionFromDataItem

From OpenStreetMap Wiki
Jump to: navigation, search

This module is used for {{KeyDescription}} and {{ValueDescription}} templates.

It operates as a pass-through module -- it takes whatever parameters were specified on the KeyDescription or ValueDescription templates, compares them with the values stored in the data items, modifies parameters as needed, and passes them on to the {{Description}} template. It also adds a few maintenance categories to make it easier to find some issues.

Please help translate it here.

Useful Queries
Number of key and tag descriptions per language

☑Y All tests passed.

Name Expected Actual
☑Y test_english
☑Y test_french
☑Y test_german
☑Y test_polish
☑Y test_polish_group
☑Y test_portuguese 
local getArgs = require('Module:Arguments').getArgs
local titleParser = require('Module:OsmPageTitleParser')
local data = mw.loadData('Module:DescriptionFromDataItem/data')
local i18n = data.translations
local p = {}

-- USEFUL DEBUGGING:
--   =p.dbg{title='Key:bridge:movable'}
--   =p.dbg{title='Key:bridge:movable', status='accepted'}
--   =p.dbg{title='Tag:noexit=yes'}
--   =p.dbg{qid='Q104'}
--   =p.dbg{qid='Q888'}
--   =p.stmt(p.GROUP, 'Q501', 'en')
--   =p.stmt(p.STATUS, 'Q5846')  -- status with ref

--   =mw.text.jsonEncode(mw.wikibase.getBestStatements('Q104', p.IMAGE), mw.text.JSON_PRETTY)
--   mw.log(mw.text.jsonEncode(stmt, mw.text.JSON_PRETTY))


-- ##########################################################################
--                             CONSTANTS
-- ##########################################################################

-- "fallback" - if this property is not set on the Tag item, check the corresponding Key item
-- "qid" - for item values, output item's Q ID
-- "en" - for item values, output english label
-- "map" - converts claim's value to the corresponding value in the given map
p.INSTANCE_OF = { id = 'P2', qid = true }
p.STATUS = { id = 'P6', en = true, fallback = true }
p.STATUS_REF = { id = 'P11' }
p.GROUP = { id = 'P25', fallback = true }
p.IMAGE = { id = 'P4' }
p.IMAGE_OSM = { id = 'P28' }
p.RENDER_IMAGE = { id = 'P38', fallback = true }
p.RENDER_IMAGE_OSM = { id = 'P39', fallback = true }
p.Q_EXCEPT = { id = 'P27' }
p.Q_LIMIT = { id = 'P26' }
p.KEY_ID = { id = 'P16', fallback = true }
p.TAG_ID = { id = 'P19', qid = true }
p.TAG_KEY = { id = 'P10' }
p.REL_ID = { id = 'P41' }
p.REL_TAG = { id = 'P40' }
p.ROLE_REL = { id = 'P43' }
p.WIKIDATA = { id = 'P12' }

local use_on_values = {
    Q8000 = 'yes',
    Q8001 = 'no',
}

local instance_types = {
    Q7 = { type = 'key', templatename = 'Template:KeyDescription' },
    Q2 = { type = 'value', templatename = 'Template:ValueDescription' },
    Q6 = { type = 'relation', templatename = 'Template:RelationDescription' },
}

p.USE_ON_NODES = { id = 'P33', fallback = true, map = use_on_values }
p.USE_ON_WAYS = { id = 'P34', fallback = true, map = use_on_values }
p.USE_ON_AREAS = { id = 'P35', fallback = true, map = use_on_values }
p.USE_ON_RELATIONS = { id = 'P36', fallback = true, map = use_on_values }

-- Makes it possible to override by unit tests
p.trackedLanguages = data.trackedLanguages

-- ##########################################################################
--                                   UTILITIES
-- ##########################################################################

local function startswith(self, str)
    return self:sub(1, #str) == str
end

local formatKeyVal = function(key, value)
    if value then
        return key .. '=' .. value
    else
        return key
    end
end

-- Normalizes yes/no/maybe into "yes", "no", nil
local function normalizeBoolean(val)
    if val then
        val = string.lower(val)
        if val == 'yes' or val == 'no' then
            return val
        end
    end
    return nil
end

local function localize(key, langCode, params)
    local msgTable = i18n[key]
    local msg
    if msgTable then
        msg = msgTable[langCode] or msgTable['en']
    end
    if not msg then
        return '<' .. key .. '>'
    end
    return mw.message.newRawMessage(msg, unpack(params or {})):plain()
end

local function getItemValue(self, prop)
    -- Only get the first returned value, so need an extra local var step
    local value = p.getClaimValue(prop, self.langCode, self.entity, self.fallbackEntity)
    return value
end

-- Format as an edit link. Target is either a relative url that starts with a slash, or an item ID (e.g. Q104)
local function editLink(self, target, msgKey)
    local file
    if msgKey == 'desc_edit_mismatch_page' then
        file = 'Red pencil.svg'
    else
        file = 'Arbcom ru editing.svg'
    end
    if not startswith(target, '/') then
        target = 'Item:' .. target
    end
    return ('&nbsp;<span class=wb-edit-pencil>[[File:' .. file .. '|12px|' ..
            localize(msgKey, self.langCode) .. '|link=' .. target .. ']]</span>')
end

-- ##########################################################################
--                              DATA ITEM PARSING
-- ##########################################################################

-- p.Q_LIMIT  "limited to region qualifier"  if qualifier is present, include the statement
--      only if self.region equals any of the listed regions
-- p.Q_EXCEPT "excluding region qualifier"   if qualifier is present, include the statement
--      only if self.region does not equal all of the listed regions
local regionQualifiers = { { prop = p.Q_LIMIT, include = true }, { prop = p.Q_EXCEPT, include = false } }

-- Test if qualifiers indicate that current statement should be
-- included or excluded based on the rules table
-- Returns true/false if it should be included, and true/false if it was based on qualifiers
local function allowRegion(region, statement)
    if statement.rank ~= 'preferred' and statement.rank ~= 'normal' then
        return false, false
    end
    local qualifiers = statement.qualifiers
    if qualifiers then
        for _, value in pairs(regionQualifiers) do
            local qualifier = qualifiers[value.prop.id]
            if qualifier then
                local include = not value.include
                for _, q in pairs(qualifier) do
                    if region == q.datavalue.value.id then
                        include = value.include
                    end
                end
                -- return after the first found rule, because multiple rules
                -- do not make any sense on the same statement
                return include, true
            end
        end
    end
    return true, false -- by default, the statement should be included
end

-- Convert claim value into a string
-- property object specifies what value to get:
--  'qid' - returns data item id
--  'map' - use a map to convert qid into a string
--  'en'  - only english label
--   default - first try local, then english, then qid
local function claimToValue(datavalue, prop, langCode)
    local result = false
    if datavalue.type == 'wikibase-entityid' then
        local qid = datavalue.value.id
        if prop.map then
            result = prop.map[qid]
        elseif not prop.qid then
            if not prop.en then
                result = p.wbGetLabelByLang(qid, langCode)
            end
            if not result then
                result = p.wbGetLabel(qid)
            end
        end
        if not result then
            result = qid
        end
    elseif datavalue.type == 'string' then
        result = datavalue.value
    else
        -- TODO:  handle other property types
        result = "Unknown datatype " .. datavalue.type
    end
    return result
end

-- Debug:  =p.getClaimValue(p.GROUP, 'en', mw.wikibase.getEntity('Q501'))
function p.getClaimValue(prop, langCode, entity, fallbackEntity)
    local usedFallback = false
    local region = data.regions[langCode]
    local statements = entity:getAllStatements(prop.id)
    if fallbackEntity and prop.fallback and next(statements) == nil then
        usedFallback = true
        statements = fallbackEntity:getAllStatements(prop.id)
    end
    local match
    for _, stmt in pairs(statements) do
        local include, qualified = allowRegion(region, stmt)
        if include then
            match = stmt
            if qualified then
                -- Keep non-qualified statement until we look through all claims,
                -- if we see a qualified one (limited to the current region), we found the best match
                break
            end
        end
    end

    local result
    local ref_link
    if match then
        -- Get reference if available
        if match.references then
            for _, ref in pairs(match.references) do
                local snak = ref.snaks[p.STATUS_REF.id]
                if snak and snak[1] then
                    ref_link = snak[1].datavalue.value
                    break
                end
            end
        end
        result = claimToValue(match.mainsnak.datavalue, prop, langCode)
    end

    return result, usedFallback, ref_link
end

local function validateKeyValue(self)
    if self.args.type == 'relation' then
        -- Ignore for relations
        return
    end
    -- Ensure key and value are set properly in the template params
    local args = self.args
    local tag = getItemValue(self, p.TAG_ID)
    local eKey, eValue = titleParser.splitKeyValue(tag)
    if not eKey then
        eKey = getItemValue(self, p.KEY_ID)
    end

    if not args.key then
        args.key = eKey
    end
    if not args.value then
        args.value = eValue
    end
    if args.key ~= eKey or args.value ~= eValue then
        table.insert(self.categories, 'Mismatched Key or Value')
    end
end

-- Get categories string, e.g. "[[category:xx]][[category:yy]]"
local function getCategories(self)
    if next(self.categories) ~= nil then
        local sortkey = formatKeyVal(self.key, self.value)
        local prefix = '[[Category:'
        local suffix = sortkey and '|' .. sortkey .. ']]' or ']]'
        return prefix .. table.concat(self.categories, suffix .. prefix) .. suffix
    else
        return nil
    end
end

local function formatValue(self, value, editLinkRef)
    if not editLinkRef then
        return value
    else
        return value .. editLink(self, editLinkRef, 'desc_edit')
    end
end

-- Process a single property, comparing old and new values
-- add tracking categories as needed
-- if qid is set, shows a pencil icon next to this value
local function processValue(self, argname, entityVal, pageVal, qid)
    local args = self.args
    if pageVal == nil then
        pageVal = args[argname]
    end
    if pageVal == '' then
        pageVal = nil
    end
    if entityVal == '' then
        entityVal = nil
    end
    if not pageVal then
        if entityVal then
            -- value is only present in the entity
            args[argname] = formatValue(self, entityVal, qid)
        else
            -- value is not set in template nor in the entity
            args[argname] = nil
        end
    elseif not entityVal then
        -- value has not been copied to the entity yet
        table.insert(self.categories, 'Not copied ' .. argname)
        args[argname] = formatValue(self, pageVal, qid)
    elseif entityVal == pageVal or
            (argname ~= 'description' and
                    self.language:caseFold(entityVal) == self.language:caseFold(pageVal)) then
        -- value is identical in both entity and the page
        -- comparison is case-insensitive except for the description

        -- For now, do not track this -- there are too many of them.
        -- Once we start cleaning them up, uncomment this tracking category
        -- table.insert(self.categories, 'Redundant ' .. argname)
        args[argname] = formatValue(self, pageVal, qid)
    else
        -- value in the page and in the entity do not match
        table.insert(self.categories, 'Mismatched ' .. argname)
        if not qid then
            args[argname] = pageVal
        else
            -- Format when value differs between Wiki and Wikibase, with a pencil
            -- For now, show pageVal with two red pencils: a red one to wiki page and gray one to Wikibase
            local editPageLink = editLink(self, self.currentTitle:fullUrl('action=edit'), 'desc_edit_mismatch_page')
            local editItemLink = editLink(self, qid, 'desc_edit_mismatch_item')

            local span = mw.html.create('span')
            span:attr('title', localize('desc_mismatch', self.langCode, { entityVal }))
                :wikitext(pageVal)
            args[argname] = tostring(span) .. editPageLink .. editItemLink

            -- In the future, switch to showing mismatched old value as red, with an edit link
            -- to the wiki page, plus new value with an edit link to Wikibase
            -- :attr('style', 'color:red')
            -- args[argname] = tostring(span) .. editPageLink .. '<br>' .. entityVal .. editItemLink
        end
    end
end

-- Get two properties from either the entity or from the fallback entity
-- Make sure they both come from the same one. Get first available value.
local function getConsistentClaims(self, prop1, prop2)
    local vals1, fb1 = p.getClaimValue(prop1, self.langCode, self.entity, self.fallbackEntity)
    local vals2, fb2 = p.getClaimValue(prop2, self.langCode, self.entity, self.fallbackEntity)
    if fb1 ~= fb2 then
        vals1 = fb1 and nil or vals1
        vals2 = fb2 and nil or vals2
    end
    return vals1 or vals2
end

local function processEntity(self)
    local args = self.args
    local qid = self.entity:getId()

    validateKeyValue(self)

    -- Compare all known parameters against the data item entity
    processValue(self, 'description',
            self.entity:getDescription(self.langCode) or self.entity:getDescription(),
            args.description, qid) -- add edit links to description

    processValue(self, 'group', getItemValue(self, p.GROUP))

    -- For status we must use english label (special processing inside the template)
    local value, usedFb, ref = p.getClaimValue(p.STATUS, self.langCode, self.entity, self.fallbackEntity)
    processValue(self, 'status', value)
    processValue(self, 'statuslink', ref)

    -- For images, first try p.IMAGE (commons image), and if not set, use p.IMAGE_OSM (osm image as string)
    local img = getConsistentClaims(self, p.IMAGE, p.IMAGE_OSM)
    processValue(self, 'image', img and 'File:' .. img or nil)

    -- Handle onRelation, onArray, onWay, and onNode
    processValue(self, 'onNode', getItemValue(self, p.USE_ON_NODES), normalizeBoolean(args.onNode))
    processValue(self, 'onWay', getItemValue(self, p.USE_ON_WAYS), normalizeBoolean(args.onWay))
    processValue(self, 'onArea', getItemValue(self, p.USE_ON_AREAS), normalizeBoolean(args.onArea))
    processValue(self, 'onRelation', getItemValue(self, p.USE_ON_RELATIONS), normalizeBoolean(args.onRelation))

    processValue(self, 'wikidata', getItemValue(self, p.WIKIDATA))
end

local function constructor(args)
    local self = {
        categories = {},
        args = args,
    }

    if args.currentTitle then
        self.currentTitle = mw.title.new(args.currentTitle)
    else
        self.currentTitle = mw.title.getCurrentTitle()
    end

    -- sets self.key and self.value from the current title
    titleParser.parseTitleToObj(self, self.currentTitle)
    if args.lang and mw.language.isSupportedLanguage(args.lang) then
        -- Override the language returned by titleParser.parseTitleToObj()
        self.language = mw.getLanguage(args.lang)
    end
    self.langCode = self.language:getCode()

    toSitelink = function(key, value)
        return (value and 'Tag:' or 'Key:') .. formatKeyVal(key, value)
    end

    local entity
    local typeGuess
    if args.qid then
        entity = p.wbGetEntity(args.qid)
    elseif args.key then
        -- template caller gave a key param (with optional value)
        entity = p.wbGetEntity(p.wbGetEntityIdForTitle(toSitelink(args.key, args.value)))
        self.key = args.key
        self.value = args.value
        typeGuess = args.value and 'Q7' or 'Q2'
    elseif args.type then
        -- template caller gave type param, guessing a relation
        entity = p.wbGetEntity(p.wbGetEntityIdForTitle('Relation:' .. args.type))
        args.key = 'type'
        args.value = args.type
        args.rtype = args.type
        args.type = 'relation'
        typeGuess = 'Q6'
    elseif args.currentTitle then
        -- template caller gave currentTitle param (probably debugging)
        entity = p.wbGetEntity(p.wbGetEntityIdForTitle(args.currentTitle))
    else
        entity = p.wbGetEntity()
        -- If there is no associated entity, try to deduce it from the title (e.g. translated pages)
        -- note that we cannot guess relations the same way
        if not entity and self.key then
            entity = p.wbGetEntity(p.wbGetEntityIdForTitle(toSitelink(self.key, self.value)))
        end
    end

    -- Try to get a fallback entity - key for tag, tag for relation, relation for rel role
    self.entity = entity
    if entity then
        local _, fbStmt = next(entity:getBestStatements(p.TAG_KEY.id))
        if not fbStmt then
            _, fbStmt = next(entity:getBestStatements(p.REL_TAG.id))
            if not fbStmt then
                _, fbStmt = next(entity:getBestStatements(p.ROLE_REL.id))
            end
        end
        if fbStmt then
            self.fallbackEntity = p.wbGetEntity(fbStmt.mainsnak.datavalue.value.id)
        end
    else
        table.insert(self.categories, 'Missing data item')
    end

    local instance_of = entity and getItemValue(self, p.INSTANCE_OF)
    local types = instance_types[instance_of or typeGuess]
    if types then
        if not args.templatename then
            args.templatename = types.templatename
        end
        if not args.type then
            args.type = types.type
        end
        if instance_of == 'Q6' then
            -- Relations are tricky - ther remap "temp" to "rtemp" and "value"
            if not args.rtype then
                args.rtype = getItemValue(self, p.REL_ID)
            end
            if not args.value then
                args.value = args.rtype
            end
        end
    end
    -- Template:Description needs these to properly format language bar and template links
    --   templatename = Template:ValueDescription | ...
    --   type = key|value|relation
    if not args.templatename then
        args.templatename = mw.getCurrentFrame():getParent():getTitle()
    end
    if not args.type then
        if args.templatename and string.find(args.templatename, 'KeyDescription', 1, true) then
            args.type = 'key'
        elseif args.templatename and string.find(args.templatename, 'ValueDescription', 1, true) then
            args.type = 'value'
        elseif args.templatename and string.find(args.templatename, 'RelationDescription', 1, true) then
            args.type = 'relation'
        end
    end
    if args.onClosedWay then
        table.insert(self.categories, 'Obsolete description template parameters')
    end

    return self
end

-- ##########################################################################
--                                 ENTRY POINTS
-- ##########################################################################

-- If we found data item for this key/tag, compare template parameters
-- with what we have in the item, and add some extra formatting/links/...
-- If the values are not provided, just use the ones from the data item
function p.main(frame)
    local args = getArgs(frame)

    -- initialize self - parse title, language, and get relevant entities
    local self = constructor(args)

    if self.entity then
        processEntity(self)
    end

    if self.entity then
        -- If this is an English item, check if description is set for
        -- the tracked languages, and if not, add a tracking category
        if self.langCode == 'en' then
            for _, lng in ipairs(p.trackedLanguages) do
                if not self.entity:getDescription(lng) then
                    table.insert(self.categories,
                            'Item with no description in language ' .. mw.ustring.upper(lng))
                end
            end
        elseif not self.entity:getDescription('en') then
            table.insert(self.categories, 'Item with no description in language EN')
        end
    end

    -- Create a group category. Use language-prefixed name if category page exists
    if self.args.group then
        local group = self.language:ucfirst(self.args.group)
        local prefix = titleParser.langPrefix(self.langCode)
        local title = mw.title.new('Category:' .. prefix .. group)
        if title.exists then
            table.insert(self.categories, title.text)
        else
            table.insert(self.categories, group)
        end
    end

    local categories = getCategories(self)

    local baseTemplate = args.basetemplate or 'Template:Description'
    if args.debuglua then
        -- debug and unit test support
        return {
            template = baseTemplate,
            args = args,
            categories = categories,
        }
    end

    local result = frame:expandTemplate { title = baseTemplate, args = args }
    if categories then
        result = result .. categories
    end
    if args.debugargs then
        result = result ..
                '<br><pre>' ..
                mw.text.nowiki(mw.text.jsonEncode(args, mw.text.JSON_PRETTY)) ..
                '</pre><br>'
    end
    return result
end


-- Create a table row to describe a specific value
-- Usually rendered as key | value | element | comment | rendering | photo
function p.row(frame)
    local args = getArgs(frame)

    if args[1] and not args.key then
        args.key = args[1]
    end
    if args[2] and not args.value then
        args.value = args[2]
    end

    -- Unlike sidecard, table could be used in different types of pages, and should not rely on auto-guessing
    assert(args.key, 'Missing key=... parameter')
    assert(args.value, 'Missing value=... parameter')

    -- initialize self - parse title, language, and get relevant entities
    local self = constructor(args)

    if self.entity then
        local qid = self.entity:getId()

        validateKeyValue(self)

        -- Compare all known parameters against the data item entity
        processValue(self, 'description',
                self.entity:getDescription(self.langCode) or self.entity:getDescription(),
                args.description, qid) -- add edit links to description

        -- For images, first try p.IMAGE (commons image), and if not set, use p.IMAGE_OSM (osm image as string)
        local photo = getConsistentClaims(self, p.IMAGE, p.IMAGE_OSM)
        processValue(self, 'photo', photo and 'File:' .. photo or nil)

        local render = getConsistentClaims(self, p.RENDER_IMAGE, p.RENDER_IMAGE_OSM)
        processValue(self, 'render', render and 'File:' .. render or nil)

        -- Handle onRelation, onArray, onWay, and onNode
        processValue(self, 'onNode', getItemValue(self, p.USE_ON_NODES), normalizeBoolean(args.onNode))
        processValue(self, 'onWay', getItemValue(self, p.USE_ON_WAYS), normalizeBoolean(args.onWay))
        processValue(self, 'onArea', getItemValue(self, p.USE_ON_AREAS), normalizeBoolean(args.onArea))
        processValue(self, 'onRelation', getItemValue(self, p.USE_ON_RELATIONS), normalizeBoolean(args.onRelation))
    end

    local elems = {}
    if 'yes' == args.onNode then
        table.insert(elems, 'iconNode')
    end
    if 'yes' == args.onWay then
        table.insert(elems, 'iconWay')
    end
    if 'yes' == args.onArea then
        table.insert(elems, 'iconArea')
    end
    if 'yes' == args.onRelation then
        table.insert(elems, 'iconRelation')
    end

    args.render = args.render and '[[' .. args.render .. '|100px]]'
    args.photo = args.photo and '[[' .. args.photo .. '|100px]]'

    if args.description2 then
        args.description = args.description .. '<br>' .. args.description2
    end

    -- key | value | element | comment | render | photo
    local resultTbl = {
        args.key,
        args.value,
        elems,
        args.description or '',
        args.render or '',
        args.photo or '',
    }
    local categories = getCategories(self)
    if args.debuglua then
        -- debug and unit test support
        return { args = args, categories = categories, row = resultTbl }
    end

    -- expand templates
    local lang = self.langCode
    for i, v in ipairs(elems) do
        elems[i] = frame:expandTemplate { title = v }
    end
    local result = '|-\n| ' .. table.concat({
        frame:expandTemplate { title = 'TagKey/exists', args = { resultTbl[1], lang = lang } },
        frame:expandTemplate { title = 'TagValue/exists', args = { resultTbl[1], resultTbl[2], lang = lang } },
        table.concat(elems, ''),
        resultTbl[4],
        resultTbl[5],
        resultTbl[6],
    }, '\n| ')
    if categories then
        result = result .. categories
    end
    if args.debugargs then
        result = result ..
                '<br><pre>' ..
                mw.text.nowiki(mw.text.jsonEncode(resultTbl, mw.text.JSON_PRETTY)) ..
                '</pre><br><pre>' ..
                mw.text.nowiki(result) ..
                '</pre><br>'
    end
    return result
end

-- ##########################################################################
--                        DEBUGGING AND TESTING SUPPORT
-- ##########################################################################
-- From the debug console, use   =p.dbg{title='Key:bridge'}
function p.dbg(args)
    args.currentTitle = args.title or 'Key:bridge:movable'
    args.debuglua = args.debuglua == nil and true or args.debuglua
    local frame = mw.getCurrentFrame():newChild { title = 'Module:DescriptionFromDataItem', args = args }
    return mw.text.jsonEncode(p.main(frame), mw.text.JSON_PRETTY)
end

-- From the debug console, use   =p.dbgrow{key='bridge', value='movable'}
function p.dbgrow(args)
    local frame = mw.getCurrentFrame():newChild { title = 'Module:DescriptionFromDataItem', args = args }
    return p.row(frame)
end

-- Debug helper for statements
-- =p.stmt(p.GROUP, 'Q501', 'en')
function p.stmt(prop, id, lang)
    return mw.text.jsonEncode(
            { p.getClaimValue(prop, lang, mw.wikibase.getEntity(id)) },
            mw.text.JSON_PRETTY)
end

-- These methods could be overwritten by unit tests
function p.wbGetEntity(entity)
    return mw.wikibase.getEntity(entity)
end

function p.wbGetEntityIdForTitle(title)
    return mw.wikibase.getEntityIdForTitle(title)
end

function p.wbGetLabel(qid)
    return mw.wikibase.getLabel(qid)
end

function p.wbGetLabelByLang(qid, langCode)
    return mw.wikibase.getLabelByLang(qid, langCode)
end

return p
Retrieved from "http://wiki.openstreetmap.org/w/index.php?title=Module:DescriptionFromDataItem&oldid=1764280"