Create Datamaps for scattered csv files

This guide demonstrates how to create individual datamaps for each Assay based on CSV tables located within the dataset folder.

Use-case

Researchers often need to store specific metadata for given data entities. Examples for this include measurement units for a particular analyte or textual descriptions of the measured properties. In the context of an ARC, we store this information within datamaps. This workflow facilitates the automated generation of these datamaps for selected Assays. For each header we want to annotate, we define a Data Context, containing Explication, Object Type and a Unit. We then map over the data files in the given assays and create a Datamap according to the headers we encounter.

Procedure

Load dependencies: Initialize and load the required ARC dependencies.

FSharp

1
#r "nuget: ARCtrl.QueryModel, 3.0.0-alpha.4"
2
open System.IO
3
open ARCtrl
4
open ARCtrl.QueryModel
5

6
let arcPath = @"H:\User\ARCs\BeechThinning"
7
let arc = ARC.load arcPath
8
let arcTables = arc.ArcTables

Define ontology terms: Specify the ontology terms corresponding to the units and explications required for the datamap.

FSharp

1
let µg_p_mg = OntologyAnnotation(
2
    name="microgram per milligram"
3
)
4
let mg_p_smm = OntologyAnnotation(
5
    name="milligram per square millimiter"
6
)
7
let unitless = OntologyAnnotation(
8
    name="none",
9
    tsr="NCIT",
10
    tan="http://purl.obolibrary.org/obo/NCIT_C41132"
11
)
12
let floatType = OntologyAnnotation(
13
    name="float",
14
    tsr="NCIT",
15
    tan="http://purl.obolibrary.org/obo/NCIT_C48150"
16
)
17
let maxQuantEfficiency = OntologyAnnotation(
18
    name="maximal quantum efficiency"
19
)
20
let quantumYield = OntologyAnnotation(
21
    name="quantum yield",
22
    tsr="TO",
23
    tan="http://purl.obolibrary.org/obo/TO_0012004"
24
)
25
let nonphotochemicalQuenching = OntologyAnnotation(
26
    name="nonphotochemical quenching",
27
    tsr="GO",
28
    tan="http://purl.obolibrary.org/obo/GO_0010196"
29
)
30
let nonregulatedQuenching = OntologyAnnotation(
31
    name="nonregulated quenching"
32
)
33
let specificeafWeight = OntologyAnnotation(
34
    name="specific leaf weight",
35
    tsr="TO",
36
    tan="http://purl.obolibrary.org/obo/TO_0001122"
37
)
38
let sleafChloCo = OntologyAnnotation(
39
    name="leaf chlorophyll content",
40
    tsr="TO",
41
    tan="http://purl.obolibrary.org/obo/TO_0012002"
42
)
43
let chloACo = OntologyAnnotation(
44
    name="chlorophyll a content",
45
    tsr="TO",
46
    tan="http://purl.obolibrary.org/obo/TO_0000293"
47
)
48
let chloBCo = OntologyAnnotation(
49
    name="chlorophyll B content",
50
    tsr="TO",
51
    tan="http://purl.obolibrary.org/obo/TO_0000295"
52
)
53
let chloRatio = OntologyAnnotation(
54
    name="chlorophyll ratio",
55
    tsr="TO",
56
    tan="http://purl.obolibrary.org/obo/TO_0000298"
57
)
58
let carContent = OntologyAnnotation(
59
    name="carotenoid content",
60
    tsr="TO",
61
    tan="http://purl.obolibrary.org/obo/TO_0000496"
62
)
63
let protPerLeafArea = OntologyAnnotation(
64
    name="Protein per leaf area"
65
)
66
let protContent = OntologyAnnotation(
67
    name="protein content",
68
    tsr="TO",
69
    tan="http://purl.obolibrary.org/obo/TO_0000598"
70
)

Optional: Add or remove DataContext values according to your specific experimental requirements.

FSharp

1
type AssayColumn =
2
    {
3
        Header : string
4
        Unit : OntologyAnnotation
5
        Description : string
6
        Explication : OntologyAnnotation
7
        ObjectType: OntologyAnnotation
8
    }
9
type AssayDefinition =
10
    {
11
        AssayName : string
12
        Columns : AssayColumn list
13
    }

Define header names: Specify the exact column headers you intend to utilize as metadata labels.

FSharp

1
let excelHeaders =
2
    [|
3
        "Specific_weight"
4
        "2:Y (II)"
5
        "2:Y (NO)"
6
        "2:Y (NPQ)"
7
        "2:Fv/Fm"
8
        "Chl_total"
9
        "Chl_a"
10
        "Chl_b"
11
        "Chl_a/b"
12
        "Car"
13
    |]

Configure Assay parameters: Define the target Assays alongside their associated metadata parameters, which include Headers (Labels), Units, Description, Explication, and ObjectType. (Note: If you modify the AssayColumn type, ensure these parameters are updated accordingly).

FSharp

1
let assays =
2
    [
3
        {
4
            AssayName = "Photosynthesis"
5
            Columns =
6
                [
7
                    {
8
                        Header = "Fv/Fm"
9
                        Unit = unitless
10
                        Description = "maximal quantum efficiency"
11
                        Explication = maxQuantEfficiency
12
                        ObjectType = floatType
13
                    }
14
                    {
15
                        Header = "Y(II)"
16
                        Unit = unitless
17
                        Description = "quantum yield"
18
                        Explication = quantumYield
19
                        ObjectType = floatType
20
                    }
21
                    {
22
                        Header = "Y(NPQ)"
23
                        Unit = unitless
24
                        Description = "nonphotochemical quenching"
25
                        Explication = nonphotochemicalQuenching
26
                        ObjectType = floatType
27
                    }
28
                    {
29
                        Header = "Y(NO)"
30
                        Unit = unitless
31
                        Description = "nonregulated quenching"
32
                        Explication = nonregulatedQuenching
33
                        ObjectType = floatType
34
                    }
35
                ]
36
        }
37
        {
38
            AssayName = "Pigment content"
39
            Columns =
40
                [
41
                    {
42
                        Header = "Chl_total"
43
                        Unit = µg_p_mg
44
                        Description = "Total chlorophyll content per leaf fresh weight"
45
                        Explication = sleafChloCo
46
                        ObjectType = floatType
47
                    }
48
                    {
49
                        Header = "Chl_a"
50
                        Unit = µg_p_mg
51
                        Description = "Chlorophyll a content per leaf fresh weight"
52
                        Explication = chloACo
53
                        ObjectType = floatType
54
                    }
55
                    {
56
                        Header = "Chl_b"
57
                        Unit = µg_p_mg
58
                        Description = "Chlorophyll b content per leaf fresh weight"
59
                        Explication = chloBCo
60
                        ObjectType = floatType
61
                    }
62
                    {
63
                        Header = "Chl_a/b"
64
                        Unit = unitless
65
                        Description = "Carotenoid content per leaf fresh weight"
66
                        Explication = chloRatio
67
                        ObjectType = floatType
68
                    }
69
                    {
70
                        Header = "Car"
71
                        Unit = µg_p_mg
72
                        Description = "Carotenoid content"
73
                        Explication = carContent
74
                        ObjectType = floatType
75
                    }
76
                ]
77
        }
78

79
        {
80
            AssayName = "LeafBiomass"
81
            Columns =
82
                [
83
                    {
84
                        Header = "Specific_weight"
85
                        Unit = mg_p_smm
86
                        Description = "Leaf fresh weight per leaf area"
87
                        Explication = specificeafWeight // Hinweis: In deiner Definition fehlt hier das 'l' bei "specificLeafWeight"
88
                        ObjectType = floatType
89
                    }
90
                ]
91
        }
92

93
        {
94
            AssayName = "Total protein content"
95
            Columns =
96
                [
97
                    {
98
                        Header = "Protein_per_Area"
99
                        Unit = mg_p_smm
100
                        Description = "Total leaf protein content per leaf area"
101
                        Explication = protPerLeafArea
102
                        ObjectType = floatType
103
                    }
104
                    {
105
                        Header = "Protein_content"
106
                        Unit = µg_p_mg
107
                        Description = "Total leaf protein content per leaf fresh weight"
108
                        Explication = protContent
109
                        ObjectType = floatType
110
                    }
111
                ]
112
        }
113
    ]

Optional: Perform any necessary string formatting or header corrections using the normalizeHeaders function.

FSharp

1
// header correction function
2
let normalizeHeader (header:string) =
3
    header
4
        .Trim()
5
        .Replace("2:", "")
6
        .Replace(" ", "")
7

8
// helper function for next step
9
let readHeaders (csvFile:string) =
10
    File.ReadLines(csvFile)
11
    |> Seq.head
12
    |> fun line ->
13
        line.Split(',')
14
        |> Array.map normalizeHeader

Generate datamaps: Execute the createDatamap function. This function iterates through the configured Assays and computationally generates the designated datamaps for each one.

FSharp

1
let createDataContexts (arcPath:string) (assay:AssayDefinition) =
2

3
    let datasetFolder =
4
        Path.Combine(arcPath, "assays", assay.AssayName, "dataset")
5

6
    if Directory.Exists datasetFolder then
7

8
        Directory.GetFiles(datasetFolder, "*.csv", SearchOption.AllDirectories)
9
        |> Array.collect (fun csvFile ->
10

11
            printfn "analyse file: %s" csvFile
12

13
            let headers = readHeaders csvFile
14

15
            let relativePath =
16
                Path.GetRelativePath(arcPath, csvFile)
17
                    .Replace("\\", "/")
18

19
            let parentFolderName = Directory.GetParent(csvFile).Name
20

21
            assay.Columns
22
            |> List.choose (fun col ->
23

24
                let wantedHeader = normalizeHeader col.Header
25

26
                headers
27
                |> Array.tryFindIndex ((=) wantedHeader)
28
                |> Option.map (fun i ->
29

30
                    let dataName = $"{relativePath}#col={i+1}"
31

32
                    printfn "  -> Match found: %s (Column %i)" wantedHeader (i+1)
33
                    DataContext(
34
                        name = dataName,
35
                        format = "text/csv",
36
                        selectorFormat = "https://datatracker.ietf.org/doc/html/rfc7111",
37
                        explication = col.Explication,
38
                        unit = col.Unit,
39
                        ObjectType = Some col.ObjectType,
40
                        label = col.Header,
41
                        description = col.Description
42

43
                    )
44
                )
45
            )
46
            |> List.toArray
47
        )
48
        |> Array.toList
49
    else
50
        printfn "directory not found: %s" datasetFolder
51
        []
52

53

54
for assay in assays do
55
    let dataContexts =
56
        createDataContexts arcPath assay
57

58
    printfn "%s: %i columns found"
59
        assay.AssayName
60
        dataContexts.Length
61

62
    if not dataContexts.IsEmpty then
63

64
        let datamap =
65
            Datamap(ResizeArray dataContexts)
66
        let isaAssay =
67
            arc.GetAssay(assay.AssayName)
68
        isaAssay.Datamap <- Some datamap
69

70
arc.Update(arcPath)