Feature/parlamint turkey

.vscode/launch.json

@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "django: runserver",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/backend/manage.py",
             "args": ["runserver"],
@@ -15,7 +15,7 @@
         },
         {
             "name": "django: shell",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/backend/manage.py",
             "args": ["shell"],
@@ -24,13 +24,22 @@
         },
         {
             "name": "django: index",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/backend/manage.py",
             "args": ["index", "${input:corpusName}"],
             "django": true,
             "justMyCode": true
         },
+        {
+            "name": "django: loadcorpora",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/backend/manage.py",
+            "args": ["loadcorpora"],
+            "django": true,
+            "justMyCode": true
+        },
         {
             "type": "chrome",
             "request": "launch",

backend/corpora/parliament/clarin_parlamint/description/parlamint_turkiye.md

@@ -0,0 +1,24 @@
+# Corpus of parliamentary debates ParlaMint-TR
+
+- Country: TR (Turkey)
+- Language tr (Turkish)
+- Version: 4.1
+- Handle: [http://hdl.handle.net/11356/1912](http://hdl.handle.net/11356/1912)
+
+
+## Documentation
+
+### Parlamint Project
+ParlaMint, a CLARIN flagship project, resulted in the creation of comparable corpora of parliamentary debates of 29 European countries and autonomous regions, covering at least the period from 2015 to 2022, and containing over 1 billion words. The corpora are uniformly encoded, contain rich metadata about their 24 thousand speakers, and are linguistically annotated up to the level of Universal Dependencies syntax and named entities.
+
+### Characteristics of the national parliament
+
+The current parliamentary system in Turkey is a unicameral system (there have been periods of bicameral systems in the past). The political system is a multi-party system. There are no official “political groups”, however, since 2018 parties make alliances during elections, which may affect their relations in the parliament as well. The current version of the corpus contains transcripts of debates in the last four terms (from 24 to 27) of the Grand National Assembly of Turkey (Turkish: Türkiye Büyük Millet Meclisi), which is approximately 50 million words recorded in  1341 sessions from June 2011 to December 2022.
+
+### Data source and acquisition
+
+The data is scraped from the official web page of the parliament (https://www.tbmm.gov.tr/). The transcripts for this period are published as HTML documents. The data is downloaded using GNU wget, and extracted from the HTML files using Beautiful Soup, and lxml libraries (Python). Except for the default processing built into these libraries (for encoding correction, HTML cleanup), no other preprocessing was applied.
+
+### Corpus-specific metadata
+
+Current version of the corpus contains the sex, the date/place of birth, and when available Wikipedia, Wikidata and Twitter linksfor regular speakers. The corpus also encodes the constituencies of the parliament members. Changes to party affiliations are documented during the time period of the corpus. Otherwise, the start of party affiliation is left unspecified.

backend/corpora/parliament/clarin_parlamint/parlamint_turkiye.py

@@ -0,0 +1,243 @@
+import logging
+from datetime import datetime
+from glob import glob
+from os.path import join
+from bs4 import BeautifulSoup
+import re
+
+from django.conf import settings
+
+from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition
+from addcorpus.python_corpora.extract import XML, Constant, Combined, Order, Metadata, Pass
+from addcorpus.es_mappings import keyword_mapping
+from corpora.utils.constants import document_context
+from corpora.parliament.parliament import Parliament
+from corpora.parliament.utils.parlamint_v4 import extract_all_org_data, extract_people_data, person_attribute_extractor, current_party_id_extractor, organisation_attribute_extractor, POLITICAL_ORIENTATIONS, node_is_current
+import corpora.parliament.utils.field_defaults as field_defaults
+
+
+from ianalyzer_readers.xml_tag import Tag
+
+logger = logging.getLogger('indexing')
+
+
+def get_persons_metadata(directory):
+    with open(join(directory, 'ParlaMint-TR-listPerson.xml'), 'rb') as f:
+        soup = BeautifulSoup(f.read(), 'xml')
+    return extract_people_data(soup)
+
+def get_orgs_metadata(directory):
+    with open(join(directory, 'ParlaMint-TR-listOrg.xml'), 'rb') as f:
+        soup = BeautifulSoup(f.read(), 'xml')
+    return extract_all_org_data(soup)
+
+def transform_political_orientation(full_string):
+    if full_string and '#orientation.' in full_string:
+        return POLITICAL_ORIENTATIONS[full_string.split('.')[1]]
+    else:
+        return None
+
+def transform_parliamentary_role(data):
+    org_nodes, date = data
+    for node in org_nodes:
+        if node['ref'] == '#TBMM' and node['role'] == 'member' and node_is_current(node, date):
+            return 'MP'
+
+def transform_ministerial_role(data):
+    org_nodes, date = data
+    for node in org_nodes:
+        if '#mstr.' in node['ref'] and node['role'] == 'head' and node_is_current(node, date):
+            for child_node in node.children:
+                if child_node.name == 'roleName' and child_node.get('xml:lang') == 'en':
+                    return child_node.text.strip()
+
+def transform_speaker_constituency(data):
+    org_nodes, date = data
+    for node in org_nodes:
+        if node['ref'] == '#TBMM' and node['role'] == 'member' and node_is_current(node, date):
+            if "#constituency-TR." in node['ana']:
+                return node['ana'].split('#constituency-TR.')[1] if node['ana'].split('#constituency-TR.')[1] else None
+            else:
+                return 'Constituency unknown'
+
+class ParlamintTurkiye(Parliament, XMLCorpusDefinition):
+    '''
+    Corpus definition for indexing Turkish parliamentary data from the ParlaMint dataset.
+    '''
+
+    title = "ParlaMint Türkiye"
+    description = "Speeches and debates from Türkiye's parliament."
+    min_date = datetime(year=2011, month=6, day=1)
+    max_date = datetime(year=2022, month=12, day=31)
+    data_directory = settings.PARLAMINT_TURKIYE_DATA
+    es_index = getattr(settings, 'PARLAMINT_TURKIYE_ES_INDEX', 'parlamint-turkiye')
+    image = 'turkiye.jpg'
+    description_page = 'parlamint_turkiye.md'
+
+    tag_toplevel = Tag('TEI')
+    tag_entry = Tag('u')
+    languages = ['tr']
+
+    category = 'parliament'
+    document_context = document_context()
+
+    def sources(self, start, end):
+        # First collect metadata that is applicable to the whole dataset, like people and parties
+        persons_metadata = get_persons_metadata(self.data_directory)
+        orgs_metadata = get_orgs_metadata(self.data_directory)
+        metadata = {
+            'persons': persons_metadata,
+            'organisations': orgs_metadata,
+        }
+
+        ## Then collect metadata that is applicable to the current file and find the paths to each xml file
+        for year in range(start.year, end.year):
+            for xml_file in glob('{}/{}/*.xml'.format(self.data_directory, year)):
+                metadata['date'] = re.search(r"\d{4}-\d{2}-\d{2}", xml_file).group()
+                yield xml_file, metadata
+
+    country = field_defaults.country()
+    country.extractor = Constant(
+        value='Türkiye'
+    )
+
+    date = field_defaults.date()
+    date.extractor = XML(
+            Tag('teiHeader'),
+            Tag('fileDesc'),
+            Tag('sourceDesc'),
+            Tag('bibl'),
+            Tag('date'),
+            toplevel=True
+    )
+
+    debate_id = field_defaults.debate_id()
+    debate_id.extractor = XML(
+            attribute='xml:id',
+            toplevel=True,
+    )
+
+    speech = field_defaults.speech(language='tr')
+    speech.extractor = XML(
+            Tag('seg'),
+            multiple=True,
+            flatten=True,
+            transform='\n'.join)
+
+    speech_id = field_defaults.speech_id()
+    speech_id.extractor = XML(
+        attribute='xml:id'
+    )
+
+    sequence = field_defaults.sequence()
+    sequence.extractor = Order(transform=lambda value: value + 1)
+
+    speaker = field_defaults.speaker()
+    speaker.extractor = person_attribute_extractor('name')
+
+    speaker_id = field_defaults.speaker_id()
+    speaker_id.extractor = person_attribute_extractor('id')
+
+    speaker_gender = field_defaults.speaker_gender()
+    speaker_gender.extractor = person_attribute_extractor('gender')
+
+    speaker_birth_year = field_defaults.speaker_birth_year()
+    speaker_birth_year.extractor = person_attribute_extractor('birth_year')
+
+    speaker_birthplace = field_defaults.speaker_birthplace()
+    speaker_birthplace.extractor = person_attribute_extractor('birthplace')
+
+    speaker_wikimedia = FieldDefinition(
+        name = 'speaker_wikimedia',
+        display_name= 'Speaker Wikipedia',
+        display_type='url',
+        description='URL to Wikimedia page of the speaker',
+        es_mapping=keyword_mapping(),
+        searchable=False,
+    )
+
+    speaker_twitter = FieldDefinition(
+        name = 'speaker_twitter',
+        display_name= 'Speaker Twitter',
+        display_type='url',
+        description='URL to Twitter page of the speaker',
+        es_mapping=keyword_mapping(),
+        searchable=False,
+    )
+
+    parliamentary_role = field_defaults.parliamentary_role()
+    parliamentary_role.extractor = Combined(
+        person_attribute_extractor('org_nodes'),
+        Metadata('date'),
+        transform=transform_parliamentary_role
+    )
+    ministerial_role = field_defaults.ministerial_role()
+    ministerial_role.extractor = Combined(
+        person_attribute_extractor('org_nodes'),
+        Metadata('date'),
+        transform=transform_ministerial_role
+    )
+
+    current_party_id = field_defaults.party_id()
+    current_party_id.extractor = current_party_id_extractor()
+
+    current_party = field_defaults.party()
+    current_party.extractor = organisation_attribute_extractor('name')
+
+    current_party_full = field_defaults.party_full()
+    current_party_full.extractor = organisation_attribute_extractor('full_name')
+
+    current_party_wiki = FieldDefinition(
+        name='party_wiki_url',
+        display_name='Wikimedia URL',
+        display_type='url',
+        description='URL to Wikimedia page of the party',
+        es_mapping=keyword_mapping(),
+        searchable=False,
+    )
+    current_party_wiki.extractor = organisation_attribute_extractor('wikimedia')
+
+    current_party_political_orientation = FieldDefinition(
+        name='political_orientation',
+        display_name='Political Orientation',
+        description="Political leaning according to the ParlaMint team",
+        es_mapping=keyword_mapping(),
+        searchable=False
+    )
+    current_party_political_orientation.extractor = Pass(
+        organisation_attribute_extractor('political_orientation'),
+        transform=transform_political_orientation
+    )
+
+    speaker_constituency = field_defaults.speaker_constituency()
+    speaker_constituency.extractor = Combined(
+        person_attribute_extractor('org_nodes'),
+        Metadata('date'),
+        transform=transform_speaker_constituency
+    )
+
+    def __init__(self):
+        self.fields = [
+            self.debate_id,
+            self.country,
+            self.date,
+            self.speech,
+            self.speech_id,
+            self.sequence,
+            self.speaker,
+            self.speaker_id,
+            self.speaker_gender,
+            self.speaker_birth_year,
+            self.speaker_birthplace,
+            self.speaker_wikimedia,
+            self.speaker_twitter,
+            self.parliamentary_role,
+            self.ministerial_role,
+            self.current_party_id,
+            self.current_party,
+            self.current_party_full,
+            self.current_party_wiki,
+            self.current_party_political_orientation,
+            self.speaker_constituency
+        ]
+

backend/corpora/parliament/clarin_parlamint/wm/documentation.md

@@ -0,0 +1 @@
+ParlaMint Turkiye does not currently include a word model.