View Javadoc
1   /*
2    * Copyright 2021 TiKV Project Authors.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    *
8    * http://www.apache.org/licenses/LICENSE-2.0
9    *
10   * Unless required by applicable law or agreed to in writing, software
11   * distributed under the License is distributed on an "AS IS" BASIS,
12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13   * See the License for the specific language governing permissions and
14   * limitations under the License.
15   *
16   */
17  
18  package org.tikv.common.columnar;
19  
20  import static org.tikv.common.util.MemoryUtil.EMPTY_BYTE_BUFFER;
21  
22  import java.math.BigDecimal;
23  import java.nio.ByteBuffer;
24  import java.nio.charset.StandardCharsets;
25  import java.time.Instant;
26  import java.time.ZoneId;
27  import java.time.ZonedDateTime;
28  import java.util.Objects;
29  import org.joda.time.LocalDate;
30  import org.tikv.common.columnar.datatypes.CHType;
31  import org.tikv.common.types.AbstractDateTimeType;
32  import org.tikv.common.types.BytesType;
33  import org.tikv.common.types.DateType;
34  import org.tikv.common.util.MemoryUtil;
35  
36  public class TiBlockColumnVector extends TiColumnVector {
37    private final ByteBuffer offsets;
38    private final ByteBuffer nullMap;
39    private final ByteBuffer data;
40    private final int fixedLength;
41  
42    public TiBlockColumnVector(CHType type, ByteBuffer data, int numOfRows, int fixedLength) {
43      super(type.toDataType(), numOfRows);
44      this.data = Objects.requireNonNull(data);
45      this.nullMap = null;
46      this.offsets = null;
47      this.fixedLength = fixedLength;
48    }
49  
50    public TiBlockColumnVector(CHType type) {
51      super(type.toDataType(), 0);
52      this.data = EMPTY_BYTE_BUFFER;
53      this.nullMap = null;
54      this.offsets = null;
55      this.fixedLength = -1;
56    }
57  
58    public TiBlockColumnVector(
59        CHType type, ByteBuffer nullMap, ByteBuffer data, int numOfRows, int fixedLength) {
60      // chType -> data type
61      super(type.toDataType(), numOfRows);
62      this.nullMap = Objects.requireNonNull(nullMap);
63      this.data = Objects.requireNonNull(data);
64      this.offsets = null;
65      this.fixedLength = fixedLength;
66    }
67  
68    /** Sets up the data type of this column vector. */
69    public TiBlockColumnVector(
70        CHType type, ByteBuffer nullMap, ByteBuffer offsets, ByteBuffer data, int numOfRows) {
71      // chType -> data type
72      super(type.toDataType(), numOfRows);
73      this.offsets = Objects.requireNonNull(offsets);
74      this.nullMap = nullMap; // may be null
75      this.data = Objects.requireNonNull(data);
76      this.fixedLength = -1;
77    }
78  
79    /**
80     * Cleans up memory for this column vector. The column vector is not usable after this.
81     *
82     * <p>This overwrites `AutoCloseable.close` to remove the `throws` clause, as column vector is
83     * in-memory and we don't expect any exception to happen during closing.
84     */
85    @Override
86    public void close() {}
87  
88    /** Returns true if this column vector contains any null values. */
89    @Override
90    public boolean hasNull() {
91      if (nullMap != null) {
92        byte[] array = nullMap.array();
93        for (byte b : array) {
94          if (b != 0) return true;
95        }
96      }
97      return false;
98    }
99  
100   /** Returns the number of nulls in this column vector. */
101   @Override
102   public int numNulls() {
103     int n = 0;
104     if (nullMap != null) {
105       byte[] array = nullMap.array();
106       for (byte b : array) {
107         if (b != 0) n++;
108       }
109     }
110     return n;
111   }
112 
113   /** Returns whether the value at rowId is NULL. */
114   @Override
115   public boolean isNullAt(int rowId) {
116     return nullMap != null && nullMap.get(rowId) != 0;
117   }
118 
119   /**
120    * Returns the boolean type value for rowId. The return value is undefined and can be anything, if
121    * the slot for rowId is null.
122    */
123   @Override
124   public boolean getBoolean(int rowId) {
125     return false;
126   }
127 
128   /**
129    * Returns the byte type value for rowId. The return value is undefined and can be anything, if
130    * the slot for rowId is null.
131    */
132   @Override
133   public byte getByte(int rowId) {
134     return data.get(rowId);
135   }
136 
137   /**
138    * Returns the short type value for rowId. The return value is undefined and can be anything, if
139    * the slot for rowId is null.
140    */
141   @Override
142   public short getShort(int rowId) {
143     return data.getShort(rowId << 1);
144   }
145 
146   /**
147    * Returns the int type value for rowId. The return value is undefined and can be anything, if the
148    * slot for rowId is null.
149    */
150   @Override
151   public int getInt(int rowId) {
152     if (type instanceof DateType) {
153       return (int) getTime(rowId);
154     }
155     return data.getInt(rowId << 2);
156   }
157 
158   // returns microseconds since epoch - fields are interpreted in current default timezone
159   private long getDateTime(int rowId) {
160     long v = data.getLong(rowId << 3);
161     long ymdhms = v >>> 24;
162     long ymd = ymdhms >>> 17;
163     int day = (int) (ymd & ((1 << 5) - 1));
164     long ym = ymd >>> 5;
165     int month = (int) (ym % 13);
166     int year = (int) (ym / 13);
167 
168     int hms = (int) (ymdhms & ((1 << 17) - 1));
169     int second = hms & ((1 << 6) - 1);
170     int minute = (hms >>> 6) & ((1 << 6) - 1);
171     int hour = hms >>> 12;
172     int microsec = (int) (v % (1 << 24));
173     ZonedDateTime zdt =
174         ZonedDateTime.of(
175             year, month, day, hour, minute, second, microsec * 1000, ZoneId.systemDefault());
176     Instant instant = zdt.toInstant();
177     return instant.getEpochSecond() * 1000_000L + instant.getNano() / 1000;
178   }
179 
180   private long getTime(int rowId) {
181     long v = data.getLong(rowId << 3);
182     long ymd = v >>> 41;
183     long ym = ymd >>> 5;
184     int year = (int) (ym / 13);
185     int month = (int) (ym % 13);
186     int day = (int) (ymd & ((1 << 5) - 1));
187     LocalDate date = new LocalDate(year, month, day);
188     return ((DateType) type).getDays(date);
189   }
190   /**
191    * Returns the long type value for rowId. The return value is undefined and can be anything, if
192    * the slot for rowId is null.
193    */
194   @Override
195   public long getLong(int rowId) {
196     if (type instanceof AbstractDateTimeType) {
197       return getDateTime(rowId);
198     }
199     if (fixedLength == 1) {
200       return getByte(rowId);
201     } else if (fixedLength == 2) {
202       return getShort(rowId);
203     } else if (fixedLength == 4) {
204       return getInt(rowId);
205     } else if (fixedLength == 8) {
206       return data.getLong(rowId << 3);
207     }
208     throw new UnsupportedOperationException(
209         String.format("getting long with fixed length %d", fixedLength));
210   }
211 
212   /**
213    * Returns the float type value for rowId. The return value is undefined and can be anything, if
214    * the slot for rowId is null.
215    */
216   @Override
217   public float getFloat(int rowId) {
218     return data.getFloat(rowId * fixedLength);
219   }
220 
221   /**
222    * Returns the double type value for rowId. The return value is undefined and can be anything, if
223    * the slot for rowId is null.
224    */
225   @Override
226   public double getDouble(int rowId) {
227     return data.getDouble(rowId * fixedLength);
228   }
229 
230   /**
231    * Returns the decimal type value for rowId. If the slot for rowId is null, it should return null.
232    */
233   @Override
234   public BigDecimal getDecimal(int rowId, int precision, int scale) {
235     if (fixedLength == 4) {
236       return MemoryUtil.getDecimal32(data, rowId << 2, scale);
237     } else if (fixedLength == 8) {
238       return MemoryUtil.getDecimal64(data, rowId << 3, scale);
239     } else if (fixedLength == 16) {
240       return MemoryUtil.getDecimal128(data, rowId << 4, scale);
241     } else {
242       return MemoryUtil.getDecimal256(data, rowId * fixedLength, scale);
243     }
244   }
245 
246   private long offsetAt(int i) {
247     return i == 0 ? 0L : offsets.getLong((i - 1) << 3);
248   }
249 
250   public int sizeAt(int i) {
251     return (int)
252         (i == 0 ? offsets.getLong(0) : offsets.getLong(i << 3) - offsets.getLong((i - 1) << 3));
253   }
254 
255   /**
256    * Returns the string type value for rowId. If the slot for rowId is null, it should return null.
257    * Note that the returned UTF8String may point to the data of this column vector, please copy it
258    * if you want to keep it after this column vector is freed.
259    */
260   @Override
261   public String getUTF8String(int rowId) {
262     // FixedString case
263     if (fixedLength != -1) {
264       byte[] chars = new byte[fixedLength];
265       data.get(chars, rowId * fixedLength, fixedLength);
266       return new String(chars);
267     } else {
268       int offset = (int) offsetAt(rowId);
269       int numBytes = sizeAt(rowId) - 1;
270       byte[] chars = new byte[numBytes];
271       data.get(chars, offset, numBytes);
272       return new String(chars, StandardCharsets.UTF_8);
273     }
274   }
275 
276   /**
277    * Returns the binary type value for rowId. If the slot for rowId is null, it should return null.
278    */
279   @Override
280   public byte[] getBinary(int rowId) {
281     if (type.equals(BytesType.BLOB) || type.equals(BytesType.TINY_BLOB)) {
282       int offset = (int) offsetAt(rowId);
283       int numBytes = sizeAt(rowId) - 1;
284       byte[] ret = new byte[numBytes];
285       data.get(ret, offset, numBytes);
286       return ret;
287     } else {
288       throw new UnsupportedOperationException(
289           "get Binary for TiBlockColumnVector is not supported");
290     }
291   }
292 
293   /** @return child [[TiColumnVector]] at the given ordinal. */
294   @Override
295   protected TiColumnVector getChild(int ordinal) {
296     throw new UnsupportedOperationException("getChild is not supported for TiBlockColumnVector");
297   }
298 }