niosx/app/ingestion/CsvIngestion.scala

/*
 * Copyright (C) 2020  Prasoon Joshi
 *
 *     This program is free software: you can redistribute it and/or modify
 *     it under the terms of the GNU General Public License as published by
 *     the Free Software Foundation, either version 3 of the License, or
 *     (at your option) any later version.
 *
 *     This program is distributed in the hope that it will be useful,
 *     but WITHOUT ANY WARRANTY; without even the implied warranty of
 *     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *     GNU General Public License for more details.
 *
 *     You should have received a copy of the GNU General Public License
 *     along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package ingestion

import com.github.tototoshi.csv._
import models.MilliEntity
import utils.FileUtils
import models._
import graphql.SchemaInputObjects._
import graphql.SchemaInputObjects.ValidField._

object CsvIngestion {

  def ingestCsvWithFieldMap(filePath: String = "conf/partner-data/sample.csv",
    cw: Crosswalk,
    partner: Partner): List[MilliEntity] = {
      val csvReader = CSVReader.open(filePath)
      csvReader
        .allWithHeaders()
        .map(fm => FlatRecordToEntity[MilliEntity].map(fm, cw, partner))
  }

}

/**
 * FlatRecord is just a Map[String, String]
 */
trait FlatRecordToEntity[E <: NiosxEntity] {
  def map(r: FlatRecord, cw: Crosswalk, partner: Partner): E
}
object FlatRecordToEntity {
  def apply[E <: NiosxEntity](implicit c: FlatRecordToEntity[E]):
  FlatRecordToEntity[E] = c
  
  def instance[E <: NiosxEntity](
    f: (FlatRecord, Crosswalk, Partner) => E): FlatRecordToEntity[E] =
      new FlatRecordToEntity[E] {
        def map(r: FlatRecord, cw: Crosswalk, partner: Partner) = 
          f(r, cw, partner)
      }

  def getFieldValue(
    field: ValidField.Value,
    r: FlatRecord,
    cw: Crosswalk): List[String] = {
      val ms: List[FlatRecordMapper] = cw.filter(_.field == field)
      ms.map({ m: FlatRecordMapper =>
        r.getOrElse(m.columnName, m.default)})
  }

  def getOptionalValue(
    field: AllowedField,
    r: FlatRecord,
    cw: Crosswalk): Option[String] = { 
      //TODO: Fix this to accept default values as well.
      val frm: Option[FlatRecordMapper] = cw.find(_.field == field)
      frm.flatMap({ m: FlatRecordMapper =>
        Some(r.getOrElse(m.columnName, m.default))
      })
  }

  implicit val me: FlatRecordToEntity[MilliEntity] =
    instance[MilliEntity]({(r,cw, partner) => 
      new MilliEntity(
        graphId        = FileUtils.getNewGraphId,
        agencyCode     = getFieldValue(ValidField.ME_agencyCode, r, cw).last,
        recordId       = getFieldValue(ValidField.ME_recordId,r, cw).last,
        unitId         = getFieldValue(ValidField.ME_unitId, r, cw).last,
        title          = getFieldValue(ValidField.ME_title, r, cw).last,
        creator        = getFieldValue(ValidField.ME_creator, r, cw).last,
        dateOfCreation = getFieldValue(ValidField.ME_dateOfCreation, r, cw).last,
        extent         = getFieldValue(ValidField.ME_extent, r, cw).last,
        level          = getFieldValue(ValidField.ME_level, r, cw).last,
        partner        = partner,
        description    = getOptionalValue(ME_description, r, cw),
        location       = getOptionalValue(ME_location, r, cw),
        accessRestrict = getOptionalValue(ME_accessRestrict, r, cw),
        useRestrict    = getOptionalValue(ME_useRestrict, r, cw),
        language       = getOptionalValue(ME_language, r, cw),
        unitType       = getOptionalValue(ME_unitType, r, cw),
        format         = getOptionalValue(ME_format, r, cw),
        images         = 
          getFieldValue(ValidField.ME_image, r, cw)
            .map({ src: String => 
            Image(
              graphId = FileUtils.getNewGraphId,
              src     = src,
              size    = ImageSize.MEDIUM)
            }),
        subjects       = 
          getFieldValue(ValidField.ME_subject, r, cw)
            .map({ subject: String => 
              val graphId = FileUtils.getNewGraphId
              Subject(
                graphId = graphId,
                prefLabel = subject,
                inScheme = "https://api.milli.link/terms",
                id = "https://api.milli.link/authority/" + graphId
              )
            })
          )
    })
}


//TODO: Include these in the MilliEntity model
//  images: [Image!]