Skip to content

Load

Loader

Class for loading data.

Methods

from_txt: Reads texts from a text file.
from_xml: Reads texts from an XML file.
forward: Filters the texts based on the target words and the maximum number of documents.
Source code in semantics/data/data_loader.py 
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
class Loader():
    """
    Class for loading data.

    Methods
    -------
        from_txt: Reads texts from a text file.
        from_xml: Reads texts from an XML file.
        forward: Filters the texts based on the target words and the maximum number of documents.
    """
    def __init__(
            self,
            texts: List[str]
            ):
        """
        Args:
            texts (List[str]): List of texts.

        Attributes:
            texts (List[str]): List of texts.
        """

        self.texts = texts


    @classmethod
    def from_txt(cls,
                 path: Union[str, Path]
                 ):
        """
        Class method to read texts from a text file.

        Args:
            path (Union[str, Path]): Path to the text file.
        """
        return cls(read_txt(path))

    @classmethod
    def from_xml(cls,
                 path: Union[str, Path],
                 tag: str
                 ):
        """
        Class method to read texts from an XML file.

        Args:
            path (Union[str, Path]): Path to the XML file.
            tag (str): Tag of the XML file to extract the texts from.
        """
        size = os.path.getsize(path)
        if size > 1e8:
            raise ValueError("File size is too large. Please split the file into smaller files.")
        tree = ET.parse(path)
        root = tree.getroot()
        texts = []
        for elem in root.findall('.//' + tag):
            if isinstance(elem.text, str):
                texts.append(elem.text)
        return cls(texts)



    def forward(
            self, 
            target_words: Optional[Union[List[str], str]] = None, 
            max_documents: Optional[int] = None, 
            shuffle: bool = True, 
            random_seed: Optional[int] = None
            ) -> List[str]:
        """
        Filters the texts based on the target words and the maximum number of documents.

        Args:
            target_words (List[str], str, None): List of target words. Defaults to None.
            max_documents (int, None): Maximum number of documents. Defaults to None.
            shuffle (bool): Whether to shuffle the data. Defaults to True.
            random_seed (int, None): Random seed. Defaults to None.

        Returns:
            texts (List[str]): List of texts.


        Examples:
            >>> from semantics.data.data_loader import Loader
            >>> texts = ['This is a test.', 'This is another test.', 'This is a third test.']
            >>> print('Original texts: ', texts)
            >>> print('Filtered texts: ', Loader(texts).forward(target_words=['third'], max_documents=1, shuffle=False))
            Original texts:  ['This is a test.', 'This is another test.', 'This is a third test.
            Filtered texts:  ['This is a third test.']
        """

        if target_words:
            relevant_texts = []
            for text in self.texts:
                if any([' ' + word + ' ' in text for word in target_words]):
                    relevant_texts.append(text)

            self.texts = relevant_texts

        if max_documents is not None:
            if shuffle:
                self.texts = sample_data(self.texts, max_documents, random_seed)
            else:
                self.texts = self.texts[:max_documents]

        return self.texts

__init__(texts)

Parameters:

Name Type Description Default
texts List[str]

List of texts.

 required

Attributes:

Name Type Description
texts List[str]

List of texts.
Source code in semantics/data/data_loader.py 
19
20
21
22
23
24
25
26
27
28
29
30
31
def __init__(
        self,
        texts: List[str]
        ):
    """
    Args:
        texts (List[str]): List of texts.

    Attributes:
        texts (List[str]): List of texts.
    """

    self.texts = texts

forward(target_words=None, max_documents=None, shuffle=True, random_seed=None)

Filters the texts based on the target words and the maximum number of documents.

Parameters:

Name Type Description Default
target_words (List[str], str, None)

List of target words. Defaults to None.

 None
max_documents (int, None)

Maximum number of documents. Defaults to None.

 None
shuffle bool

Whether to shuffle the data. Defaults to True.

 True
random_seed (int, None)

Random seed. Defaults to None.

 None

Returns:

Name Type Description
texts List[str]

List of texts.

Examples:

>>> from semantics.data.data_loader import Loader
>>> texts = ['This is a test.', 'This is another test.', 'This is a third test.']
>>> print('Original texts: ', texts)
>>> print('Filtered texts: ', Loader(texts).forward(target_words=['third'], max_documents=1, shuffle=False))
Original texts:  ['This is a test.', 'This is another test.', 'This is a third test.
Filtered texts:  ['This is a third test.']
Source code in semantics/data/data_loader.py 
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
def forward(
        self, 
        target_words: Optional[Union[List[str], str]] = None, 
        max_documents: Optional[int] = None, 
        shuffle: bool = True, 
        random_seed: Optional[int] = None
        ) -> List[str]:
    """
    Filters the texts based on the target words and the maximum number of documents.

    Args:
        target_words (List[str], str, None): List of target words. Defaults to None.
        max_documents (int, None): Maximum number of documents. Defaults to None.
        shuffle (bool): Whether to shuffle the data. Defaults to True.
        random_seed (int, None): Random seed. Defaults to None.

    Returns:
        texts (List[str]): List of texts.


    Examples:
        >>> from semantics.data.data_loader import Loader
        >>> texts = ['This is a test.', 'This is another test.', 'This is a third test.']
        >>> print('Original texts: ', texts)
        >>> print('Filtered texts: ', Loader(texts).forward(target_words=['third'], max_documents=1, shuffle=False))
        Original texts:  ['This is a test.', 'This is another test.', 'This is a third test.
        Filtered texts:  ['This is a third test.']
    """

    if target_words:
        relevant_texts = []
        for text in self.texts:
            if any([' ' + word + ' ' in text for word in target_words]):
                relevant_texts.append(text)

        self.texts = relevant_texts

    if max_documents is not None:
        if shuffle:
            self.texts = sample_data(self.texts, max_documents, random_seed)
        else:
            self.texts = self.texts[:max_documents]

    return self.texts

from_txt(path) classmethod

Class method to read texts from a text file.

Parameters:

Name Type Description Default
path Union[str, Path]

Path to the text file.

 required
Source code in semantics/data/data_loader.py 
34
35
36
37
38
39
40
41
42
43
44
@classmethod
def from_txt(cls,
             path: Union[str, Path]
             ):
    """
    Class method to read texts from a text file.

    Args:
        path (Union[str, Path]): Path to the text file.
    """
    return cls(read_txt(path))

from_xml(path, tag) classmethod

Class method to read texts from an XML file.

Parameters:

Name Type Description Default
path Union[str, Path]

Path to the XML file.

 required
tag str

Tag of the XML file to extract the texts from.

 required
Source code in semantics/data/data_loader.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
@classmethod
def from_xml(cls,
             path: Union[str, Path],
             tag: str
             ):
    """
    Class method to read texts from an XML file.

    Args:
        path (Union[str, Path]): Path to the XML file.
        tag (str): Tag of the XML file to extract the texts from.
    """
    size = os.path.getsize(path)
    if size > 1e8:
        raise ValueError("File size is too large. Please split the file into smaller files.")
    tree = ET.parse(path)
    root = tree.getroot()
    texts = []
    for elem in root.findall('.//' + tag):
        if isinstance(elem.text, str):
            texts.append(elem.text)
    return cls(texts)

split_xml(path, output_dir, max_children=1000)

Splits an XML file into multiple XML files with a maximum number of children.

Parameters:

Name Type Description Default
path str

Path to the XML file.

 required
output_dir str

Path to the output directory.

 required
max_children int

Maximum number of children. Defaults to 1000.

 1000

Returns:

Name Type Description
paths List[str]

List of paths to the new XML files.
Source code in semantics/data/data_loader.py 
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
def split_xml(path:str, output_dir:str, max_children:int = 1000) -> List[str]:
    """
    Splits an XML file into multiple XML files with a maximum number of children.

    Args:
        path (str): Path to the XML file.
        output_dir (str): Path to the output directory.
        max_children (int, optional): Maximum number of children. Defaults to 1000.

    Returns:
        paths (List[str]): List of paths to the new XML files.
    """

    # Parse the XML
    tree = ET.parse(path)
    root = tree.getroot()
    file_name = Path(path).stem

    paths = []
    # Create new XML trees based on the split
    for idx, i in enumerate(range(0, len(root), max_children)):
        new_root = ET.Element(root.tag, root.attrib)
        new_root.extend(root[i:i + max_children])
        new_tree = ET.ElementTree(new_root)
        new_tree.write(f"{output_dir}/{file_name}_{idx}.xml")
        paths.append(f"{output_dir}/{file_name}_{idx}.xml")

    return paths