-
Notifications
You must be signed in to change notification settings - Fork 227
snippets
Mattias Wadman edited this page Oct 24, 2023
·
20 revisions
Snippets to put in fq init.jq
etc.
# xml plist to json
# fq -L . -Rs 'include "dev/plist"; fromplist' file.plist
def from_plist:
def _f:
( . as [$n, $a, $c]
| $c
| if $n == "dict" then
( chunk(2)
| map({key: (.[0] | _f), value: (.[1] | _f)})
| from_entries
)
elif $n == "array" then map(_f)
elif $n == "true" then true
elif $n == "false" then false
else $a."#text"
end
);
fromxml({array: true}) | .[2][0] | _f;
# to_tar({filename: "a", data: "aaa"}, {filename: "b", data: "bbb"}) | tar
# fq -L . 'include "tar"; to_tar(foreach grep_by(format=="jpeg") as $i (0; .+1; {filename: "image\(.).jpeg", data: $i}))' file | tar xv
def to_tar(g):
def lpad($l; $n): [[range($l-length) | $n], .];
def rpad($l; $n): [., [range($l-length) | $n]];
def header($filename; $b):
def checksum: [.[range(.size)]] | add;
def h:
[ ($filename | rpad(100; 0)) # name
, ("000644 " | rpad(8; 0)) # mode
, ("000000 " | rpad(8; 0)) # uid
, ("000000 " | rpad(8; 0)) # gid
, [($b.size | to_radix(8) | [lpad(11; "0")]), " "] # size
, [(0| to_radix(8) | lpad(11; "0")), " "] # mtime
, " " # chksum (blank spaces when adding checksum)
, ("0") # typeflag
, ("" | rpad(100; 0)) # linkname
, ["ustar", 0] # magic
, ("00") # version
, ("user" | rpad(32; 0)) # uname
, ("group" | rpad(32; 0)) # gname
, ("000000 " | rpad(8; 0)) # devmajor
, ("000000 " | rpad(8; 0)) # devminor
, ("" | rpad(155; 0)) # prefix
] | tobytes;
( h as $h
| [ $h[0:148]
, [(($h | checksum) | to_radix(8) | lpad(6; "0")), 0, " "]
, $h[148+8:]
]
| tobytes
);
[ ( # per file
( g as {$filename, $data}
| ($data | tobytes) as $b
| ($filename | rpad(100; 0)) # name
| header($filename; $b) as $header
| $header
, ("" | lpad((512 - ($header.size % 512)) % 512; 0))
, $b
, ("" | lpad((512 - ($b.size % 512)) % 512; 0))
)
# end_marker
, [range(1024) | 0]
)
] | tobytes;
# "01:02:03.45" -> 3723.45
def from_duration:
( reduce (split(":") | reverse[]) as $p (
{m: 1, n: 0};
( .n = .n + ($p | tonumber) * .m
| .m *= 60
)
)
| .n
);
# 3723.45 -> "01:02:03.45"
def to_duration:
def intfloor: (. % (.+1));
def intdiv($n): (. - (. % $n)) / $n;
def pad($s): $s[length:] + .;
( intfloor as $n # int floor
| (tostring | split(".")[1]) as $frac # hack: use string for decimal fractions
| $n
| [ recurse(if . > 0 then intdiv(60) else empty end)
| . % 60
| tostring
| pad("00")
]
| reverse[1:]
| join(":") + if $frac then "." + $frac else "" end
);
# similar to https://github.com/tomnomnom/gron
def gron:
( path(..) as $p
| getpath($p)
| scalars
| "\($p | path_to_expr) = \(tojson)"
| println
);
def _xml_entities:
{ "<": "<",
">": ">",
"&": "&",
"\"": """,
"'": "'"
};
# NOTE: later version of fq has builtin support for xml
def xml_entity_encode:
def _f($t):
gsub(
"(?<c>[^\\w ])";
"\(.c | . as $c | if $t | has($c) then $t[.] else explode[0] | "&#\(.);" end)"
);
_f(_xml_entities);
def xml_data_encode:
def _f($t):
gsub(
"(?<c>[<>&\"''])";
"\($t[.c])"
);
_f(_xml_entities);
# ["tag"] -> <tag/>
# ["tag", {k: v, ...}] -> <tag k="v" .../>
# ["tag", {k: v, ...}, [["tag", ...], "text", ...]] -> <tag k="v" ...><tag/> text...</tag>
def to_xml($indent; $nl):
def _f($d):
def _attr: to_entries | map("\(.key)=\(.value | tostring | xml_entity_encode | tojson)") | join(" ");
if .[2] then
( "\($d)<\(.[0]) \(.[1] | _attr)>"
, (.[2][] | if type == "array" then _f($d+$indent) else xml_data_encode end)
, "\($d)</\(.[0])>"
)
elif .[1] then "\($d)<\(.[0]) \(.[1] | _attr)/>"
else "\($d)<\(.[0])/>"
end;
( [_f("")]
| join($nl)
);
def toxml($indent): toxml($indent; "\n");
def toxml: toxml(" ");
# [[pixel, pixel, ...], ...] | tobmp
# all rows needs to have same amount of pixels
def to_bmp:
def le16: [band(.; 255), band(bsr(.; 8); 255)];
def le24: [band(.; 255), band(bsr(.; 8); 255), band(bsr(.; 16); 255)];
def le32: [band(.; 255), band(bsr(.; 8); 255), band(bsr(.; 16); 255), band(bsr(.; 24); 255)];
( (.[0] | length) as $width
| length as $height
| ( [ (40 | le32)
, ($width | le32)
, ($height | le32)
, (1 | le16) # planes
, (24 | le16) # bits per pixel
, (0 | le32) # no compression
, (($width*3 + (($width*3) % 4)) * $height | le32) # byte size of raw pixels including padding
, (0 | le32) # horizontal p/m
, (0 | le32) # vertical p/m
, (0 | le32) # no colors in palette
, (0 | le32) # no important colors (?)
]
| tobytes
) as $dib_header
| ( reverse | map(map(le24) | [., [range(($width*3) % 4) | 0]]) # rrggbb, ... -> [[[bb,gg,rr], ...], padding]
| tobytes
) as $raw_pixels
| [ "BM" # bmp magic
, (14 + $dib_header.size + $raw_pixels.size | le32) # size of file
, (0 | le16) # app specific unused0
, (0 | le16) # app specific unused1
, (14 + $dib_header.size | le32)
, $dib_header
, $raw_pixels
] | tobytes
);
# use iterm control codes to display an image
def display_image: "\x1b]1337;File=inline=1:\(base64)\x07" | println;
# swedish_flag_bmp | display_image
def swedish_flag_bmp:
def r($n;$v): range($n) | $v;
( [0x006aa7,0xfecc02] as [$b,$g]
| [r(100;$b),r(50;$g),r(200;$b)] as $a
| [r(350;$g)] as $b
| [r(100;$a),r(50;$b),r(100;$a)]
| tobmp
);
# convert buffer to bmp with width $w
# usage: tobytes | buffer_to_pixels(500) | display_image
def buffer_to_pixels($w):
( ((.size - (.size%$w)) / $w) as $r
| [.[range($r*$w)]]
| chunk($w)
| tobmp
);
# see all first frame where sample_rate changes
# ex: .frames | changes(.header.sample_rate)
def changes(f): streaks_by(f)[].[0];
def urldecode:
gsub(
"%(?<c>[a-fA-F0-9]{2})";
( .c
| ascii_downcase
| explode
# "0"-"9" or "a"-"f"
| map(.-48 | if .>=49 then .-39 else . end)
| [.[0]*16+.[1]]
| implode
)
);
# converted from https://github.com/FFmpeg/FFmpeg/blob/870bfe16a12bf09dca3a4ae27ef6f81a2de80c40/libavutil/display.c
# usage: fgrep("matrix_structure") | (parent | mp4_path), mp4_matrix_structure_rotation
def mp4_matrix_structure_rotation:
( .a as $s0
| .c as $s3
| .b as $s1
| .d as $s4
| ($s0*$s0 + $s3*$s3 | sqrt) as $scale0
| ($s1*$s1 + $s4*$s4 | sqrt) as $scale1
| atan2($s1/$scale1; $s0 / $scale0) * 180 / 3.14159265359
| -round
);
# hack to parse just a box
# NOTE: in new versions of you can do mp4({force: true}) instead
# <binary> | mp4_box
def mp4_box:
[0, 0, 0, 16, "ftyp", "isom", 0, 0 , 2 , 0, .] | mp4.boxes;