seppo/mcdb/t_mcdb.ml

(*
 *    _  _   ____                         _  
 *  _| || |_/ ___|  ___ _ __  _ __   ___ | | 
 * |_  ..  _\___ \ / _ \ '_ \| '_ \ / _ \| | 
 * |_      _|___) |  __/ |_) | |_) | (_) |_| 
 *   |_||_| |____/ \___| .__/| .__/ \___/(_) 
 *                     |_|   |_|             
 *
 * Personal Social Web.
 *
 * Copyright (C) The #Seppo contributors. All rights reserved.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *)

open Alcotest
open Mcdb__

let set_up () =
  Unix.chdir "../../../test/"

let tc_hash () =
  "s" |> Bytes.of_string |> Ds_cdb.hash
  |> check int32  __LOC__ 177622l;

  "s" |> Bytes.of_string |> Mcdb.hash32_byt |> Optint.to_int |> check int __LOC__ 177622;

  "a" |> Bytes.of_string |> Mcdb.hash32_byt |> Optint.to_int |> check int __LOC__  0x2b5c4 ;
  let k0 =
    "http://www.traunsteiner-tagblatt.de/region+lokal/landkreis-traunstein/traunstein/pressemitteilungen-der-stadt-traunstein_artikel,-Traunstein-20-%E2%80%93-Neue-Medien-im-Mittelpunkt-_arid,198374.html"
  in
  k0 |> Bytes.of_string |> Mcdb.hash32_byt |> Optint.to_int64 |> check int64 __LOC__ 0xc7410a37L ;

  assert true

let tc_hash_foldr () =
  "s"
  |> Astring.String.fold_left Mcdb.H.foldr Mcdb.H._32_5381
  |> Optint.to_int |> check int __LOC__ 177622;
  "s"
  |> String.to_bytes |> Bytes.fold_left Mcdb.H.foldr Mcdb.H._32_5381
  |> Optint.to_int |> check int __LOC__ 177622;
  "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
  |> Astring.String.fold_left Mcdb.H.foldr Mcdb.H._32_5381
  |> Optint.to_string
  |> check string __LOC__ "1316135747"

let tc_find_string_opt () =
  (* Logr.info (fun m -> m "mapcdb_test.test_find_string_opt"); *)
  Mcdb.find_string_opt "s" (Cdb "mini.cdb")
  |> Option.get
  |> check string __LOC__ "ß";
  assert (None = Mcdb.find_string_opt "zzz" (Cdb "mini.cdb"));
(*
  find_string_opt
    "http://www.traunsteiner-tagblatt.de/region+lokal/landkreis-traunstein/traunstein/pressemitteilungen-der-stadt-traunstein_artikel,-Traunstein-20-%E2%80%93-Neue-Medien-im-Mittelpunkt-_arid,198374.html"
    (Cdb "big.cdb")
  |> Option.get |> String.length
  |> check int __LOC__ 1000;
*)
  assert true

let pDir = 0o755

let mk_db fn =
  (try
     Unix.mkdir "tmp" pDir;
   with Unix.Unix_error (Unix.EEXIST, _, _) -> ());
  (try
     Unix.unlink fn;
     Unix.unlink (fn ^ "~")
   with Unix.Unix_error (Unix.ENOENT, _, _) -> ());
  Mcdb.Cdb fn

let _tc_update () =
  (* Logr.info (fun m -> m "mapcdb_test.test_update"); *)
  let fn = mk_db "tmp/add.cdb" in
  let _ = Mcdb.update_string "a" "1" fn in
  Mcdb.find_string_opt "a" fn |> Option.get |> check string __LOC__ "1";
  let _ = Mcdb.update_string "a" "2" fn in
  Mcdb.find_string_opt "a" fn |> Option.get |> check string __LOC__ "2";
  let _ = Mcdb.remove_string "a" fn in
  assert (Mcdb.find_string_opt "a" fn |> Option.is_none);
  assert true

let tc_fold () =
  (* Logr.info (fun m -> m "%s.%s" "mapcdb_test" "fold_left"); *)
  let cdb = Mcdb.Cdb "data/mini.cdb" in
  Mcdb.fold_left (fun init (k,v) ->
      let k = k |> Bytes.to_string
      and v = v |> Bytes.to_string in
      (* Logr.debug (fun m -> m "  %s->%s" k v); *)
      (k,v) :: init) [] cdb
  |> List.length |> check int __LOC__ 3

let tc_fold_1 () =
  Mcdb.Cdb "data/2024-04-30-131146-subscribed.cdb"
  |> Mcdb.fold_left (fun c _ -> 1 + c) 0
  |> check int __LOC__ 52

let tc_hash32_str_base64 () =
  (* Logr.info (fun m -> m "%s.%s" "Mcdb_test" "test_hash32_str_base64"); *)
  Optint.max_int
  |> Optint.to_int64
  |> Printf.sprintf "0x%Lx"
  |> check string __LOC__ "0x7fffffff";
  Mcdb.H._32_0xFFffFFff
  |> Optint.to_int64
  |> Printf.sprintf "0x%Lx"
  |> check string __LOC__ "0xffffffff";
  let h = "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
          |> Uri.of_string |> Uri.to_string
          |> Mcdb.hash32_str in
  h |> Optint.to_string
  |> check string __LOC__ "1316135747";
  Optint.encoded_size |> check int __LOC__ 4;
  let b = Bytes.make Optint.encoded_size (Char.chr 0) in
  h |> Optint.encode b ~off:0;
  b |> Bytes.to_string
  |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
  |> check string __LOC__ "TnKjQw";
  assert true

let tc_hash63_str_base64 () =
  (* Logr.info (fun m -> m "%s.%s" "Mcdb_test" "test_hash63_str_base64"); *)
  Optint.Int63.encoded_size |> check int __LOC__ 8;
  let _mask_63 = Optint.Int63.max_int
  and _5381_63 = 5381 |> Optint.Int63.of_int in
  (* http://cr.yp.to/cdb/cdb.txt *)
  let hash63_gen len f_get : Optint.Int63.t =
    let ( +. )   = Optint.Int63.add
    and ( << )   = Optint.Int63.shift_left
    and ( ^ )    = Optint.Int63.logxor
    and ( land ) = Optint.Int63.logand in
    let rec fkt (idx : int) (h : Optint.Int63.t) =
      if idx = len
      then h
      else
        let c = idx |> f_get |> Char.code |> Optint.Int63.of_int in
        (((h << 5) +. h) ^ c) land _mask_63
        |> fkt (succ idx)
    in
    fkt 0 _5381_63
  in
  let hash63_str dat : Optint.Int63.t =
    hash63_gen (String.length dat) (String.get dat)
  in
  let uhash ?(off = 0) ?(buf = Bytes.make (Optint.Int63.encoded_size) (Char.chr 0)) u =
    u
    |> Uri.to_string
    |> hash63_str
    |> Optint.Int63.encode buf ~off;
    buf
    |> Bytes.to_string
    |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
  in
  _mask_63
  |> Optint.Int63.to_int64
  |> Printf.sprintf "0x%Lx"
  |> check string __LOC__ "0x3fffffffffffffff";
  let h = "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
          |> Uri.of_string |> Uri.to_string
          |> hash63_str in
  h |> Optint.Int63.to_string
  |> check string __LOC__ "4387560302522311491";
  let b = Bytes.make Optint.Int63.encoded_size (Char.chr 0) in
  h |> Optint.Int63.encode b ~off:0;
  b |> Bytes.to_string
  |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
  |> check string __LOC__ "POO-2U5yo0M";
  "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda" |> Uri.of_string
  |> uhash |> check string __LOC__ "POO-2U5yo0M";
  "https://digitalcourage.social/users/mro" |> Uri.of_string
  |> uhash |> check string __LOC__ "BlamH6XVcgE";
  "https://digitalcourage.social/users/mrp" |> Uri.of_string
  |> uhash |> check string __LOC__ "BlamH6XVch4";
  "ittps://digitalcourage.social/users/mrp" |> Uri.of_string
  |> uhash |> check string __LOC__ "LesfhY0ub58";
  assert true

let _bench_update_1 n =
  (* Logr.info (fun m -> m "mapcdb_test.bench_update_1"); *)
  let fn = mk_db "tmp/add_1.cdb" in
  let i = ref 1 in
  while !i < n do
    Printf.printf ".";
    let k = Printf.sprintf "%d %f" !i (Sys.time()) in
    let _ = Mcdb.update_string k k fn in
    incr i
  done;
  assert true

let _bench_update_2 n =
  (* Logr.info (fun m -> m "mapcdb_test.bench_update_2"); *)
  let fn = mk_db "tmp/add_2.cdb" in
  let l = List.init n
      (fun i ->
         let k :bytes = Printf.sprintf "%d" i |> Bytes.of_string in
         (k,k)) in
  let s = List.to_seq l in
  let all _ = true in
  let _ = Mcdb.add_seq all s fn in
  assert true

let _bench_update_3 n =
  (* Logr.info (fun m -> m "mapcdb_test.bench_update_3"); *)
  let fn = mk_db "tmp/add_3.cdb"
  and all _ = true
  and fkt add =
    let i = ref 0 in
    while !i < n do
      let k = Printf.sprintf "%d" !i |> Bytes.of_string in
      let v = Printf.sprintf "%f" (Sys.time()) |> Bytes.of_string in
      let _ = add (k,v) in
      incr i
    done;
  in
  let _ = Mcdb.add_many all fkt fn in
  assert true

let () =
  run
    "Mcdb" [
    __FILE__ , [
      "set_up"              , `Quick, set_up;
      "tc_hash"             , `Quick, tc_hash ;
      "tc_hash_fold"        , `Quick, tc_hash_foldr ;
      "tc_find_string_opt"  , `Quick, tc_find_string_opt ;
      (* "tc_update"           , `Quick, tc_update ; *)
      "tc_fold"             , `Quick, tc_fold ;
      "tc_fold_1"           , `Quick, tc_fold_1 ;
      "tc_hash32_str_base64", `Quick, tc_hash32_str_base64 ;
      "tc_hash63_str_base64", `Quick, tc_hash63_str_base64 ;
      (* "bench_update_1"        , `Quick, bench_update_1 1000; *)
      (* "bench_update_2"        , `Quick, bench_update_2 100_000; *)
      (* "bench_update_3"        , `Quick, bench_update_3 100_000; *)
    ]
  ]