seppo/test/t_mapcdb.ml

(*
 *    _  _   ____                         _  
 *  _| || |_/ ___|  ___ _ __  _ __   ___ | | 
 * |_  ..  _\___ \ / _ \ '_ \| '_ \ / _ \| | 
 * |_      _|___) |  __/ |_) | |_) | (_) |_| 
 *   |_||_| |____/ \___| .__/| .__/ \___/(_) 
 *                     |_|   |_|             
 *
 * Personal Social Web.
 *
 * Copyright (C) The #Seppo contributors. All rights reserved.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *)

open Seppo_lib

let o i = i |> Optint.of_int

let oo i = i |> Optint.of_int64

let test_hash () =
  Logr.info (fun m -> m "mapcdb_test.test_hash");
  let open Mapcdb in
  "s" |> Bytes.of_string |> Ds_cdb.hash
  |> Assrt.equals_int32 __LOC__ 177622l;

  "s" |> Bytes.of_string |> hash32_byt |> Assrt.equals_optint __LOC__ (o 177622);

  Assrt.equals_optint __LOC__ (o 0x2b5c4) ("a" |> Bytes.of_string |> hash32_byt);
  let k0 =
    "http://www.traunsteiner-tagblatt.de/region+lokal/landkreis-traunstein/traunstein/pressemitteilungen-der-stadt-traunstein_artikel,-Traunstein-20-%E2%80%93-Neue-Medien-im-Mittelpunkt-_arid,198374.html"
  in
  Assrt.equals_optint __LOC__ (oo 0xc7410a37L) (k0 |> Bytes.of_string |> hash32_byt);

  assert true

let test_find_string_opt () =
  Logr.info (fun m -> m "mapcdb_test.test_find_string_opt");
  let open Mapcdb in
  find_string_opt "s" (Cdb "mini.cdb")
  |> Option.get
  |> Assrt.equals_string __LOC__ "ß";
  assert (None = find_string_opt "zzz" (Cdb "mini.cdb"));
(*
  find_string_opt
    "http://www.traunsteiner-tagblatt.de/region+lokal/landkreis-traunstein/traunstein/pressemitteilungen-der-stadt-traunstein_artikel,-Traunstein-20-%E2%80%93-Neue-Medien-im-Mittelpunkt-_arid,198374.html"
    (Cdb "big.cdb")
  |> Option.get |> String.length
  |> Assrt.equals_int __LOC__ 1000;
*)
  assert true

let mk_db fn =
  (try
     Unix.mkdir "tmp" File.pDir;
   with Unix.Unix_error (Unix.EEXIST, _, _) -> ());
  (try
     Unix.unlink fn;
     Unix.unlink (fn ^ "~")
   with Unix.Unix_error (Unix.ENOENT, _, _) -> ());
  File.touch fn;
  Mapcdb.Cdb fn

let test_update () =
  Logr.info (fun m -> m "mapcdb_test.test_update");
  let fn = mk_db "tmp/add.cdb" in
  let _ = Mapcdb.update_string "a" "1" fn in
  Mapcdb.find_string_opt "a" fn |> Option.get |> Assrt.equals_string __LOC__ "1";
  let _ = Mapcdb.update_string "a" "2" fn in
  Mapcdb.find_string_opt "a" fn |> Option.get |> Assrt.equals_string __LOC__ "2";
  let _ = Mapcdb.remove_string "a" fn in
  assert (Mapcdb.find_string_opt "a" fn |> Option.is_none);
  assert true

let test_fold () =
  Logr.info (fun m -> m "%s.%s" "mapcdb_test" "fold_left");
  let cdb = Mapcdb.Cdb "data/mini.cdb" in
  Mapcdb.fold_left (fun init (k,v) ->
      let k = k |> Bytes.to_string
      and v = v |> Bytes.to_string in
      (* Logr.debug (fun m -> m "  %s->%s" k v); *)
      (k,v) :: init) [] cdb
  |> List.length |> Assrt.equals_int __LOC__ 3

let test_fold_1 () =
  Mapcdb.Cdb "data/2024-04-30-131146-subscribed.cdb"
  |> Mapcdb.fold_left (fun c _ -> 1 + c) 0 
  |> Assrt.equals_int __LOC__ 52

let test_hash32_str_base64 () =
  Logr.info (fun m -> m "%s.%s" "Mapcdb_test" "test_hash32_str_base64");
  Optint.max_int
  |> Optint.to_int64
  |> Printf.sprintf "0x%Lx"
  |> Assrt.equals_string __LOC__ "0x7fffffff";
  Mapcdb._32_0xFFffFFff
  |> Optint.to_int64
  |> Printf.sprintf "0x%Lx"
  |> Assrt.equals_string __LOC__ "0xffffffff";
  let h = "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
          |> Uri.of_string |> Uri.to_string
          |> Mapcdb.hash32_str in
  h |> Optint.to_string
  |> Assrt.equals_string __LOC__ "1316135747";
  Optint.encoded_size |> Assrt.equals_int __LOC__ 4;
  let b = Bytes.make Optint.encoded_size (Char.chr 0) in
  h |> Optint.encode b ~off:0;
  b |> Bytes.to_string
  |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
  |> Assrt.equals_string __LOC__ "TnKjQw";
  assert true

let test_hash63_str_base64 () =
  Logr.info (fun m -> m "%s.%s" "Mapcdb_test" "test_hash63_str_base64");
  Optint.Int63.encoded_size |> Assrt.equals_int __LOC__ 8;
  let _mask_63 = Optint.Int63.max_int
  and _5381_63 = 5381 |> Optint.Int63.of_int in
  (* http://cr.yp.to/cdb/cdb.txt *)
  let hash63_gen len f_get : Optint.Int63.t =
    let ( +. )   = Optint.Int63.add
    and ( << )   = Optint.Int63.shift_left
    and ( ^ )    = Optint.Int63.logxor
    and ( land ) = Optint.Int63.logand in
    let rec fkt (idx : int) (h : Optint.Int63.t) =
      if idx = len
      then h
      else
        let c = idx |> f_get |> Char.code |> Optint.Int63.of_int in
        (((h << 5) +. h) ^ c) land _mask_63
        |> fkt (succ idx)
    in
    fkt 0 _5381_63
  in
  let hash63_str dat : Optint.Int63.t =
    hash63_gen (String.length dat) (String.get dat)
  in
  let uhash ?(off = 0) ?(buf = Bytes.make (Optint.Int63.encoded_size) (Char.chr 0)) u =
    u
    |> Uri.to_string
    |> hash63_str
    |> Optint.Int63.encode buf ~off;
    buf
    |> Bytes.to_string
    |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
  in
  _mask_63
  |> Optint.Int63.to_int64
  |> Printf.sprintf "0x%Lx"
  |> Assrt.equals_string __LOC__ "0x3fffffffffffffff";
  let h = "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
          |> Uri.of_string |> Uri.to_string
          |> hash63_str in
  h |> Optint.Int63.to_string
  |> Assrt.equals_string __LOC__ "4387560302522311491";
  let b = Bytes.make Optint.Int63.encoded_size (Char.chr 0) in
  h |> Optint.Int63.encode b ~off:0;
  b |> Bytes.to_string
  |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
  |> Assrt.equals_string __LOC__ "POO-2U5yo0M";
  "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda" |> Uri.of_string
  |> uhash |> Assrt.equals_string __LOC__ "POO-2U5yo0M";
  "https://digitalcourage.social/users/mro" |> Uri.of_string
  |> uhash |> Assrt.equals_string __LOC__ "BlamH6XVcgE";
  "https://digitalcourage.social/users/mrp" |> Uri.of_string
  |> uhash |> Assrt.equals_string __LOC__ "BlamH6XVch4";
  "ittps://digitalcourage.social/users/mrp" |> Uri.of_string
  |> uhash |> Assrt.equals_string __LOC__ "LesfhY0ub58";
  assert true

let bench_update_1 n =
  Logr.info (fun m -> m "mapcdb_test.bench_update_1");
  let fn = mk_db "tmp/add_1.cdb" in
  let i = ref 1 in
  while !i < n do
    Printf.printf ".";
    let k = Printf.sprintf "%d %f" !i (Sys.time()) in
    let _ = Mapcdb.update_string k k fn in
    incr i
  done;
  assert true

let bench_update_2 n =
  Logr.info (fun m -> m "mapcdb_test.bench_update_2");
  let fn = mk_db "tmp/add_2.cdb" in
  let l = List.init n
      (fun i ->
         let k :bytes = Printf.sprintf "%d" i |> Bytes.of_string in
         (k,k)) in
  let s = List.to_seq l in
  let all _ = true in
  let _ = Mapcdb.add_seq all s fn in
  assert true

let bench_update_3 n =
  Logr.info (fun m -> m "mapcdb_test.bench_update_3");
  let fn = mk_db "tmp/add_3.cdb"
  and all _ = true
  and fkt add =
    let i = ref 0 in
    while !i < n do
      let k = Printf.sprintf "%d" !i |> Bytes.of_string in
      let v = Printf.sprintf "%f" (Sys.time()) |> Bytes.of_string in
      let _ = add (k,v) in
      incr i
    done;
  in
  let _ = Mapcdb.add_many all fkt fn in
  assert true

let () =
  Unix.chdir "../../../test/";
  test_hash ();
  test_find_string_opt ();
  test_update ();
  test_fold ();
  test_fold_1 ();
  test_hash32_str_base64 ();
  test_hash63_str_base64 ();
  bench_update_1 1000;
  bench_update_2 100_000;
  bench_update_3 100_000;
  assert true